diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..c48fafa --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,96 @@ +name: Release + +on: + push: + tags: + - 'v*' + workflow_dispatch: + inputs: + tag: + description: 'Tag to release (e.g., v0.1.0)' + required: true + type: string + prerelease: + description: 'Mark as pre-release' + required: false + default: false + type: boolean + +jobs: + release: + runs-on: ubuntu-latest + permissions: + contents: write + id-token: write + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install build twine + + - name: Extract version from tag + id: version + run: | + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + TAG="${{ github.event.inputs.tag }}" + else + TAG="${GITHUB_REF#refs/tags/}" + fi + VERSION=${TAG#v} + echo "tag=$TAG" >> $GITHUB_OUTPUT + echo "version=$VERSION" >> $GITHUB_OUTPUT + echo "Tag: $TAG, Version: $VERSION" + + - name: Update version in pyproject.toml + run: | + sed -i 's/version = "[^"]*"/version = "${{ steps.version.outputs.version }}"/' pyproject.toml + echo "Updated version to ${{ steps.version.outputs.version }}" + grep 'version =' pyproject.toml + + - name: Generate changelog for this release + id: changelog + run: | + if [ ! -f CHANGELOG.md ]; then + echo "# Changelog" > CHANGELOG.md + echo "" >> CHANGELOG.md + echo "All notable changes to this project will be documented in this file." >> CHANGELOG.md + echo "" >> CHANGELOG.md + echo "The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)," >> CHANGELOG.md + echo "and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html)." >> CHANGELOG.md + echo "" >> CHANGELOG.md + fi + + # Extract release notes for this version + if grep -q "## \[${{ steps.version.outputs.version }}\]" CHANGELOG.md; then + awk '/## \[${{ steps.version.outputs.version }}\]/{flag=1; next} /## \[/{flag=0} flag' CHANGELOG.md > release_notes.md + else + echo "Release ${{ steps.version.outputs.tag }}" > release_notes.md + echo "" >> release_notes.md + echo "This release includes the latest improvements and bug fixes." >> release_notes.md + fi + + - name: Create GitHub Release + uses: softprops/action-gh-release@v1 + with: + tag_name: ${{ steps.version.outputs.tag }} + name: Release ${{ steps.version.outputs.tag }} + body_path: release_notes.md + draft: false + prerelease: ${{ github.event.inputs.prerelease == 'true' }} + generate_release_notes: true + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Clean up + run: rm -f release_notes.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..9b5f3b6 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,61 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +### Added +- Automated release workflow with GitHub Actions +- Semantic versioning support +- This CHANGELOG.md file + +### Changed +- Repository now supports stable tags and releases + +## [0.1.0] - 2025-10-19 + +### Added +- NVSHMEM4Py Integration - replaced custom python bindings with official Python language binding for NVSHMEM +- Cuda Graph support +- Flexible transportation layers: NVLink, IBGDA, IBRC, EFA +- Overlapping communication and computation +- Comprehensive testing and benchmarking suite +- C++ testing support with CMake +- Multi-node testing and benchmarking capabilities + +### Changed +- Migrated from static linking of host-side initialization (libnvshmem.a) to dynamic linking (libnvshmem_host.so) +- Updated README.md with comprehensive installation and usage instructions +- Improved project structure with proper Python packaging + +### Fixed +- Fixed potential bugs in internode dispatch kernel (shared memory allocation and synchronization) +- Fixed build linkage errors +- Fixed NVSHMEM external variable instances issues +- Memory leaks in Distributed::allToAllImpl (freed srcBuffer and dstBuffer) +- Added device guard for kernel calls + +### Security +- Proper linking of NVSHMEM to sublibraries as interface to prevent multiple instances + +--- + +## Version History Overview + +This project follows [Semantic Versioning](https://semver.org/): +- **MAJOR** version for incompatible API changes +- **MINOR** version for backwards-compatible functionality additions +- **PATCH** version for backwards-compatible bug fixes + +For consumers of pplx-kernels, you can now use version tags instead of pinning to specific SHAs: + +```bash +# Install specific version +pip install git+https://github.com/perplexityai/pplx-kernels.git@v0.1.0 + +# Or in requirements.txt +git+https://github.com/perplexityai/pplx-kernels.git@v0.1.0 +``` diff --git a/README.md b/README.md index ac3bb68..2a6d6f1 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@ Features: * ✅ Cuda Graph support * ✅ Flexible transportation layers: NVLink, IBGDA, IBRC, EFA * ✅ Overlapping communication and computation +* ✅ Stable releases with semantic versioning ## System Requirements @@ -12,12 +13,42 @@ To learn how to set up the system drivers and dependencies, refer to the [Instal ## Installation +### Using Stable Releases (Recommended) + +With stable releases, you can now pin to specific versions instead of commit SHAs: + +```bash +# Install latest stable release +pip install git+https://github.com/perplexityai/pplx-kernels.git + +# Install specific version +pip install git+https://github.com/perplexityai/pplx-kernels.git@v0.1.0 +``` + +For use in `requirements.txt`: +``` +# Pin to specific stable version +git+https://github.com/perplexityai/pplx-kernels.git@v0.1.0 + +# Or use latest release +git+https://github.com/perplexityai/pplx-kernels.git +``` + +### Development Installation + +For development or building from source: + ```bash +git clone https://github.com/perplexityai/pplx-kernels.git cd pplx-kernels TORCH_CUDA_ARCH_LIST=9.0a+PTX python3 setup.py bdist_wheel pip install dist/*.whl ``` +## Versioning + +This project follows [Semantic Versioning](https://semver.org/). See [CHANGELOG.md](CHANGELOG.md) for version history and [docs/RELEASE_PROCESS.md](docs/RELEASE_PROCESS.md) for release information. + ## Single-node Testing and Benchmarking Test: @@ -120,3 +151,7 @@ To run the all-to-all benchmarks on one node: ```bash NVSHMEM_REMOTE_TRANSPORT=none mpirun -np 4 ./all_to_all/bench_all_to_all ``` + +## Contributing + +See [CHANGELOG.md](CHANGELOG.md) for version history and [docs/RELEASE_PROCESS.md](docs/RELEASE_PROCESS.md) for information about releases. diff --git a/docs/RELEASE_PROCESS.md b/docs/RELEASE_PROCESS.md new file mode 100644 index 0000000..4aa3e56 --- /dev/null +++ b/docs/RELEASE_PROCESS.md @@ -0,0 +1,171 @@ +# Release Process + +This document describes the release process for pplx-kernels, including how to create stable tags and releases. + +## Overview + +The pplx-kernels project follows [Semantic Versioning](https://semver.org/) and provides stable releases through Git tags and GitHub releases. This allows consumers to pin to specific versions instead of using commit SHAs. + +## Version Format + +Versions follow the format `vX.Y.Z` where: +- `X` = Major version (breaking changes) +- `Y` = Minor version (new features, backwards compatible) +- `Z` = Patch version (bug fixes, backwards compatible) + +Examples: `v0.1.0`, `v1.0.0`, `v1.2.3` + +## Release Types + +### 1. Automated Release (Recommended) + +Use GitHub Actions workflow for consistent releases: + +1. Go to the [Actions tab](https://github.com/perplexityai/pplx-kernels/actions) +2. Select "Release" workflow +3. Click "Run workflow" +4. Specify: + - Tag (e.g., `v0.2.0`) + - Whether it's a pre-release +5. The workflow will: + - Update `pyproject.toml` version + - Create a Git tag + - Generate release notes + - Create GitHub release + +### 2. Manual Release + +For manual releases: + +```bash +# 1. Update version in pyproject.toml +sed -i 's/version = "[^"]*"/version = "0.2.0"/' pyproject.toml + +# 2. Update CHANGELOG.md +# Add new version section with changes + +# 3. Commit changes +git add pyproject.toml CHANGELOG.md +git commit -m "Prepare release v0.2.0" + +# 4. Create and push tag +git tag -a v0.2.0 -m "Release v0.2.0" +git push origin v0.2.0 + +# 5. Create GitHub release manually or let the workflow handle it +``` + +## Pre-release Process + +For testing releases before stable versions: + +1. Use pre-release tags: `v0.2.0-alpha.1`, `v0.2.0-beta.1`, `v0.2.0-rc.1` +2. Mark as pre-release in GitHub +3. Test thoroughly before creating stable release + +## Updating CHANGELOG.md + +Before each release, update `CHANGELOG.md`: + +```markdown +## [X.Y.Z] - YYYY-MM-DD + +### Added +- New features + +### Changed +- Modifications to existing features + +### Fixed +- Bug fixes + +### Removed +- Removed features + +### Security +- Security improvements +``` + +## Installation Instructions for Users + +With stable releases, users can now install specific versions: + +### Using pip +```bash +# Latest stable release +pip install git+https://github.com/perplexityai/pplx-kernels.git + +# Specific version +pip install git+https://github.com/perplexityai/pplx-kernels.git@v0.1.0 +``` + +### Using requirements.txt +``` +# Pin to specific version +git+https://github.com/perplexityai/pplx-kernels.git@v0.1.0 + +# Use version range (when published to PyPI) +pplx-kernels>=0.1.0,<0.2.0 +``` + +### Using Poetry +```toml +[tool.poetry.dependencies] +pplx-kernels = {git = "https://github.com/perplexityai/pplx-kernels.git", tag = "v0.1.0"} +``` + +## Migration Guide + +### For Existing Users + +If you're currently pinning to commit SHAs, migrate to version tags: + +**Before:** +```bash +pip install git+https://github.com/perplexityai/pplx-kernels.git@2bd6e30fab358829bd01d1c0907be8088ff9e5e1 +``` + +**After:** +```bash +pip install git+https://github.com/perplexityai/pplx-kernels.git@v0.1.0 +``` + +## Hotfix Process + +For critical fixes that need immediate release: + +1. Create hotfix branch from the release tag +2. Apply minimal fix +3. Increment patch version (e.g., v0.1.0 → v0.1.1) +4. Follow normal release process +5. Merge back to main branch + +## Release Checklist + +- [ ] Update version in `pyproject.toml` +- [ ] Update `CHANGELOG.md` with new version +- [ ] Test build and installation locally +- [ ] Run full test suite +- [ ] Create and push Git tag +- [ ] Create GitHub release with release notes +- [ ] Verify release artifacts +- [ ] Update documentation if needed +- [ ] Announce release (if applicable) + +## Troubleshooting + +### Version Conflicts +If you encounter version conflicts, ensure `pyproject.toml` version matches the Git tag. + +### Failed Builds +Check that all dependencies are properly specified and the build environment is correct. + +### Missing Tags +Ensure tags are pushed to the remote repository: `git push origin --tags` + +## Future Improvements + +- PyPI publishing for easier installation +- Automated changelog generation from commit messages +- Integration with package managers +- Automated testing for each release diff --git a/scripts/release.py b/scripts/release.py new file mode 100644 index 0000000..e9cf56d --- /dev/null +++ b/scripts/release.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python3 +""" +Release management script for pplx-kernels. + +This script helps automate version bumping and release preparation. +""" + +import argparse +import re +import subprocess +import sys +from datetime import datetime +from pathlib import Path +from typing import Optional + + +def get_current_version() -> str: + """Get current version from pyproject.toml.""" + pyproject_path = Path("pyproject.toml") + if not pyproject_path.exists(): + msg = "pyproject.toml not found" + raise FileNotFoundError(msg) + + content = pyproject_path.read_text(encoding="utf-8") + match = re.search(r'version = "([^"]+)"', content) + if not match: + msg = "Version not found in pyproject.toml" + raise ValueError(msg) + + return match.group(1) + + +def update_version(new_version: str) -> None: + """Update version in pyproject.toml.""" + pyproject_path = Path("pyproject.toml") + content = pyproject_path.read_text(encoding="utf-8") + + updated_content = re.sub( + r'version = "[^"]+"', + f'version = "{new_version}"', + content, + ) + + pyproject_path.write_text(updated_content, encoding="utf-8") + print(f"Updated pyproject.toml version to {new_version}") # noqa: T201 + + +def update_changelog(version: str, changes: Optional[str] = None) -> None: # noqa: ARG001 + """Update CHANGELOG.md with new version.""" + changelog_path = Path("CHANGELOG.md") + if not changelog_path.exists(): + print("CHANGELOG.md not found, skipping changelog update") # noqa: T201 + return + + content = changelog_path.read_text(encoding="utf-8") + today = datetime.now().strftime("%Y-%m-%d") + + # Find the [Unreleased] section and add new version + unreleased_pattern = r'## \[Unreleased\]' + if re.search(unreleased_pattern, content): + new_section = f"## [Unreleased]\n\n## [{version}] - {today}" + content = re.sub(unreleased_pattern, new_section, content) + else: + # Add new section after the header + header_end = content.find('\n\n') + if header_end != -1: + insertion_point = header_end + 2 + new_section = f"## [{version}] - {today}\n\n" + content = content[:insertion_point] + new_section + content[insertion_point:] + + changelog_path.write_text(content, encoding="utf-8") + print(f"Updated CHANGELOG.md with version {version}") # noqa: T201 + + +def bump_version(current_version: str, bump_type: str) -> str: + """Bump version based on type (major, minor, patch).""" + parts = current_version.split('.') + if len(parts) != 3: + msg = f"Invalid version format: {current_version}" + raise ValueError(msg) + + major, minor, patch = map(int, parts) + + if bump_type == "major": + return f"{major + 1}.0.0" + if bump_type == "minor": + return f"{major}.{minor + 1}.0" + if bump_type == "patch": + return f"{major}.{minor}.{patch + 1}" + msg = f"Invalid bump type: {bump_type}" + raise ValueError(msg) + + +def create_git_tag(version: str, *, push: bool = False) -> bool: + """Create and optionally push git tag.""" + tag = f"v{version}" + + # Check if tag already exists + result = subprocess.run( + ["git", "tag", "-l", tag], + capture_output=True, + text=True, + check=False, + ) + if result.stdout.strip(): + print(f"Tag {tag} already exists") # noqa: T201 + return False + + # Create tag + subprocess.run(["git", "tag", "-a", tag, "-m", f"Release {tag}"], check=True) + print(f"Created tag {tag}") # noqa: T201 + + if push: + subprocess.run(["git", "push", "origin", tag], check=True) + print(f"Pushed tag {tag} to origin") # noqa: T201 + + return True + + +def commit_changes(version: str) -> None: + """Commit version changes.""" + files = ["pyproject.toml", "CHANGELOG.md"] + existing_files = [f for f in files if Path(f).exists()] + + if not existing_files: + print("No files to commit") # noqa: T201 + return + + subprocess.run(["git", "add", *existing_files], check=True) + subprocess.run( + ["git", "commit", "-m", f"Prepare release v{version}"], + check=True, + ) + print(f"Committed changes for version {version}") # noqa: T201 + + +def main() -> None: + """Main function for release management.""" + parser = argparse.ArgumentParser( + description="Release management for pplx-kernels", + ) + parser.add_argument( + "--version", + help="Specific version to set (e.g., 1.2.3)", + ) + parser.add_argument( + "--bump", + choices=["major", "minor", "patch"], + help="Bump version", + ) + parser.add_argument( + "--tag", + action="store_true", + help="Create git tag", + ) + parser.add_argument( + "--push", + action="store_true", + help="Push tag to origin", + ) + parser.add_argument( + "--commit", + action="store_true", + help="Commit changes", + ) + parser.add_argument( + "--current", + action="store_true", + help="Show current version", + ) + + args = parser.parse_args() + + if args.current: + print(f"Current version: {get_current_version()}") # noqa: T201 + return + + if not any([args.version, args.bump]): + parser.error("Must specify either --version or --bump") + + current_version = get_current_version() + print(f"Current version: {current_version}") # noqa: T201 + + if args.version: + new_version = args.version + else: + new_version = bump_version(current_version, args.bump) + + print(f"New version: {new_version}") # noqa: T201 + + # Update files + update_version(new_version) + update_changelog(new_version) + + # Git operations + if args.commit: + commit_changes(new_version) + + if args.tag: + create_git_tag(new_version, push=args.push) + + print("\nRelease preparation complete!") # noqa: T201 + print("Next steps:") # noqa: T201 + if not args.commit: + print( # noqa: T201 + f" 1. Review changes and commit: git add . && git commit -m 'Prepare release v{new_version}'", + ) + if not args.tag: + print(f" 2. Create tag: git tag -a v{new_version} -m 'Release v{new_version}'") # noqa: T201 + if not args.push: + print(f" 3. Push tag: git push origin v{new_version}") # noqa: T201 + print(" 4. The GitHub Actions workflow will create the release automatically") # noqa: T201 + + +if __name__ == "__main__": + main()