diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 3ab1eb59..4f0eccd6 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -55,3 +55,22 @@ jobs: with: fail_ci_if_error: false files: ./coverage.xml + + test-pipeline-generator: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.10 + uses: actions/setup-python@v2 + with: + python-version: "3.10" + - name: Install uv + uses: astral-sh/setup-uv@v6 + - name: Install dependencies + working-directory: tools/pipeline-generator + run: | + uv sync + - name: Run tests + working-directory: tools/pipeline-generator + run: | + uv run pytest diff --git a/monai/deploy/operators/dicom_data_loader_operator.py b/monai/deploy/operators/dicom_data_loader_operator.py index bc590cb6..59963433 100644 --- a/monai/deploy/operators/dicom_data_loader_operator.py +++ b/monai/deploy/operators/dicom_data_loader_operator.py @@ -437,10 +437,10 @@ def test(): print(f" 'SeriesDescription': {ds.SeriesDescription if ds.SeriesDescription else ''}") print( " 'IssuerOfPatientID':" - f" {ds.get('IssuerOfPatientID', '').repval if ds.get('IssuerOfPatientID', '') else '' }" + f" {ds.get('IssuerOfPatientID', '').repval if ds.get('IssuerOfPatientID', '') else ''}" ) try: - print(f" 'IssuerOfPatientID': {ds.IssuerOfPatientID if ds.IssuerOfPatientID else '' }") + print(f" 'IssuerOfPatientID': {ds.IssuerOfPatientID if ds.IssuerOfPatientID else ''}") except AttributeError: print( " If the IssuerOfPatientID does not exist, ds.IssuerOfPatientID would throw AttributeError." diff --git a/monai/deploy/operators/monai_bundle_inference_operator.py b/monai/deploy/operators/monai_bundle_inference_operator.py index 7ae4db4d..0395d3b8 100644 --- a/monai/deploy/operators/monai_bundle_inference_operator.py +++ b/monai/deploy/operators/monai_bundle_inference_operator.py @@ -1,4 +1,4 @@ -# Copyright 2002 MONAI Consortium +# Copyright 2022-2025 MONAI Consortium # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -13,6 +13,7 @@ import logging import os import pickle +import sys import tempfile import time import zipfile @@ -101,7 +102,11 @@ def _read_from_archive(archive, root_name: str, config_name: str, do_search=True return content_text def _extract_from_archive( - archive, root_name: str, config_names: List[str], dest_folder: Union[str, Path], do_search=True + archive, + root_name: str, + config_names: List[str], + dest_folder: Union[str, Path], + do_search=True, ): """A helper function for extract files of configs from the archive to the destination folder @@ -151,6 +156,45 @@ def _extract_from_archive( if isinstance(config_names, str): config_names = [config_names] + # Check if bundle_path is a directory (for directory-based bundles) + bundle_path_obj = Path(bundle_path) + if bundle_path_obj.is_dir(): + # Handle directory-based bundles + parser = ConfigParser() + + # Read metadata from configs/metadata.json + metadata_path = bundle_path_obj / "configs" / "metadata.json" + if not metadata_path.exists(): + raise IOError(f"Cannot find metadata.json at {metadata_path}") + + with open(metadata_path, "r") as f: + metadata_content = f.read() + parser.read_meta(f=json.loads(metadata_content)) + + # Read other config files + config_files = [] + for config_name in config_names: + config_name_base = config_name.split(".")[0] # Remove extension if present + # Validate config name to prevent path traversal + if ".." in config_name_base or "/" in config_name_base or "\\" in config_name_base: + raise ValueError(f"Invalid config name: {config_name_base}") + found = False + for suffix in bundle_suffixes: + config_path = bundle_path_obj / "configs" / f"{config_name_base}{suffix}" + if config_path.exists(): + ... + config_files.append(config_path) + found = True + break + if not found: + raise IOError(f"Cannot find config file for {config_name} in {bundle_path_obj / 'configs'}") + + parser.read_config(config_files) + parser.parse() + + return parser + + # Original ZIP file handling code name, _ = os.path.splitext(os.path.basename(bundle_path)) # bundle file name same archive folder name parser = ConfigParser() @@ -363,6 +407,10 @@ def __init__( if self._bundle_path and self._bundle_path.is_file(): self._init_config(self._bundle_config_names.config_names) self._init_completed = True + elif self._bundle_path and self._bundle_path.is_dir(): + # For directory-based bundles, delay initialization to compute method + logging.debug(f"Bundle path {self._bundle_path} is a directory. Will initialize during execution.") + # Keep the bundle_path for directory-based bundles else: logging.debug( f"Bundle, at path {self._bundle_path}, not available. Will get it in the execution context." @@ -420,6 +468,11 @@ def _init_config(self, config_names): config_names ([str]): Names of the config (files) in the bundle """ + # Ensure bundle root is on sys.path so 'scripts.*' can be imported + bundle_root = str(self._bundle_path) + if bundle_root not in sys.path: + sys.path.insert(0, bundle_root) + parser = get_bundle_config(str(self._bundle_path), config_names) self._parser = parser @@ -562,7 +615,79 @@ def compute(self, op_input, op_output, context): # When run as a MAP docker, the bundle file is expected to be in the context, even if the model # network is loaded on a remote inference server (when the feature is introduced). logging.debug(f"Model network not loaded. Trying to load from model path: {self._bundle_path}") - self._model_network = torch.jit.load(self.bundle_path, map_location=self._device).eval() + + # Check if bundle_path is a directory + if self._bundle_path.is_dir(): + # For directory-based bundles, look for model in models/ subdirectory + model_path = self._bundle_path / "models" / "model.ts" + if not model_path.exists(): + # Try model.pt as fallback + model_path = self._bundle_path / "models" / "model.pt" + if not model_path.exists(): + raise IOError(f"Cannot find model.ts or model.pt in {self._bundle_path / 'models'}") + + # Ensure device is set + if not hasattr(self, "_device"): + self._device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + # Initialize config for directory bundles if not already done + if not self._init_completed: + logging.info(f"Initializing config from directory bundle: {self._bundle_path}") + self._init_config(self._bundle_config_names.config_names) + self._init_completed = True + + # Load model based on file type + if model_path.suffix == ".ts": + # TorchScript bundle + self._model_network = torch.jit.load(str(model_path), map_location=self._device).eval() + else: + # .pt checkpoint: instantiate network from config and load state dict + try: + # Some .pt files may still be TorchScript; try jit first + self._model_network = torch.jit.load(str(model_path), map_location=self._device).eval() + except Exception: + # Fallback to eager model with loaded weights + if self._parser is None: + # Ensure parser/config are initialized + self._init_config(self._bundle_config_names.config_names) + # Instantiate network from config + # Ensure bundle root is on sys.path so 'scripts.*' can be imported + bundle_root = str(self._bundle_path) + if bundle_root not in sys.path: + sys.path.insert(0, bundle_root) + network = ( + self._parser.get_parsed_content("network") + if self._parser.get("network") is not None + else None + ) + if network is None: + # Backward compatibility: some bundles use "network_def" then to(device) + network = ( + self._parser.get_parsed_content("network_def") + if self._parser.get("network_def") is not None + else None + ) + if network is not None: + network = network.to(self._device) + if network is None: + raise RuntimeError("Unable to instantiate network from bundle configs.") from None + + checkpoint = torch.load(str(model_path), map_location=self._device) + # Determine the state dict layout + state_dict = None + if isinstance(checkpoint, dict): + if "state_dict" in checkpoint and isinstance(checkpoint["state_dict"], dict): + state_dict = checkpoint["state_dict"] + elif "model" in checkpoint and isinstance(checkpoint["model"], dict): + state_dict = checkpoint["model"] + if state_dict is None: + # Assume raw state dict + state_dict = checkpoint + network.load_state_dict(state_dict, strict=True) + self._model_network = network.eval() + else: + # Original ZIP bundle handling + self._model_network = torch.jit.load(self._bundle_path, map_location=self._device).eval() else: raise IOError("Model network is not load and model file not found.") @@ -701,7 +826,50 @@ def _receive_input(self, name: str, op_input, context): logging.debug(f"Shape of the converted input image: {value.shape}") logging.debug(f"Metadata of the converted input image: {metadata}") elif isinstance(value, np.ndarray): + # Keep numpy array as-is when possible and set metadata so downstream transforms handle channels. + # Use bundle metadata to infer expected number of channels and adjust conservatively. + ndims = value.ndim + expected_channels = None + try: + in_meta = self._inputs.get(name, {}) + if isinstance(in_meta, dict): + expected_channels = in_meta.get("num_channels") + except Exception: + expected_channels = None + + if ndims == 3: + # No channel present (W, H, D) + if expected_channels is not None and expected_channels > 1: + raise ValueError( + f"Input for '{name!r}' has no channel dimension but bundle expects {expected_channels} channels. " + "Provide multi-channel input or add a transform to stack channels before inference." + ) + # else expected 1 or unknown -> proceed without channel + elif ndims == 4: + # Channel-last assumed (W, H, D, C) + actual_channels = value.shape[-1] + if expected_channels is not None and expected_channels != actual_channels: + if expected_channels == 1 and actual_channels > 1: + logging.warning( + "Input for '%s' has %d channels but bundle expects 1; selecting channel 0.", + name, + actual_channels, + ) + value = value[..., 0] + ndims = 3 + else: + raise ValueError( + f"Input for '{name!r}' has {actual_channels} channels but bundle expects {expected_channels}." + ) + # else exact match or unknown -> keep as-is + else: + # Unsupported rank for medical image input + raise ValueError(f"Unsupported input rank {ndims} for '{name!r}'. Expected 3D (W,H,D) or 4D (W,H,D,C).") value = torch.from_numpy(value).to(self._device) + if metadata is None: + metadata = {} + # Indicate whether there was a channel for EnsureChannelFirstd + metadata["original_channel_dim"] = "no_channel" if ndims == 3 else -1 # else value is some other object from memory @@ -732,7 +900,28 @@ def _send_output(self, value: Any, name: str, metadata: Dict, op_output, context raise TypeError("arg 1 must be of type torch.Tensor or ndarray.") logging.debug(f"Output {name} numpy image shape: {value.shape}") - result: Any = Image(np.swapaxes(np.squeeze(value, 0), 0, 2).astype(np.uint8), metadata=metadata) + + # Handle 2D masks and generic 2D tensors gracefully + if value.ndim == 2: + # Already HxW image; binarize/scale left to downstream operators + out_img = value.astype(np.uint8) + result: Any = Image(out_img, metadata=metadata) + elif value.ndim == 3: + # Could be (C, H, W) with C==1 or (H, W, C) + if value.shape[0] == 1: # (1, H, W) -> (H, W) + out_img = value[0].astype(np.uint8) + result = Image(out_img, metadata=metadata) + elif value.shape[-1] == 1: # (H, W, 1) -> (H, W) + out_img = value[..., 0].astype(np.uint8) + result = Image(out_img, metadata=metadata) + else: + # Fallback to original behavior for 3D volumetric layout assumptions + out_img = np.swapaxes(np.squeeze(value, 0), 0, 2).astype(np.uint8) + result = Image(out_img, metadata=metadata) + else: + # Keep existing behavior for higher-dimensional data (e.g., 3D volumes) + out_img = np.swapaxes(np.squeeze(value, 0), 0, 2).astype(np.uint8) + result = Image(out_img, metadata=metadata) logging.debug(f"Converted Image shape: {result.asnumpy().shape}") elif otype == np.ndarray: result = np.asarray(value) diff --git a/monai/deploy/operators/nii_data_loader_operator.py b/monai/deploy/operators/nii_data_loader_operator.py index 67b0e070..d5b2bfe9 100644 --- a/monai/deploy/operators/nii_data_loader_operator.py +++ b/monai/deploy/operators/nii_data_loader_operator.py @@ -80,7 +80,24 @@ def convert_and_save(self, nii_path): image_reader = SimpleITK.ImageFileReader() image_reader.SetFileName(str(nii_path)) image = image_reader.Execute() - image_np = np.transpose(SimpleITK.GetArrayFromImage(image), [2, 1, 0]) + image_np = SimpleITK.GetArrayFromImage(image) + + # Handle different dimensionalities properly + if image_np.ndim == 3: + # Standard 3D volume: transpose from (z, y, x) to (x, y, z) + image_np = np.transpose(image_np, [2, 1, 0]) + elif image_np.ndim == 4: + # 4D volume with channels: (c, z, y, x) to (c, x, y, z) + image_np = np.transpose(image_np, [0, 3, 2, 1]) + elif image_np.ndim == 2: + # 2D slice: transpose from (y, x) to (x, y) + image_np = np.transpose(image_np, [1, 0]) + else: + # For other dimensions, log a warning and return as-is + self._logger.warning( + f"Unexpected {image_np.ndim}D NIfTI file shape {image_np.shape} from {nii_path}, returning without transpose" + ) + return image_np diff --git a/tools/pipeline-generator/.gitignore b/tools/pipeline-generator/.gitignore new file mode 100644 index 00000000..aae96ea8 --- /dev/null +++ b/tools/pipeline-generator/.gitignore @@ -0,0 +1,2 @@ +results*/ +test_*/ diff --git a/tools/pipeline-generator/README.md b/tools/pipeline-generator/README.md new file mode 100644 index 00000000..f45dbf0a --- /dev/null +++ b/tools/pipeline-generator/README.md @@ -0,0 +1,241 @@ +# Pipeline Generator + +A CLI tool for generating [MONAI Deploy](https://github.com/Project-MONAI/monai-deploy-app-sdk) application pipelines from [MONAI Bundles](https://docs.monai.io/en/stable/bundle_intro.html). + +## Features + +- List available MONAI models from HuggingFace +- Generate complete MONAI Deploy applications from HuggingFace models +- Support for multiple model sources through configuration +- Automatic bundle download and analysis +- Template-based code generation with Jinja2 +- Beautiful output formatting with Rich (Python library for rich text and beautiful formatting) + +## Platform Requirements + +- **Linux/Unix operating systems only** +- Compatible with MONAI Deploy App SDK platform support +- Ubuntu 22.04+ recommended (aligned with main SDK requirements) + +## Installation + +```bash +# Clone the repository +cd tools/pipeline-generator/ + +# Install with uv (no virtualenv needed - uv manages it per command) +uv pip install -e ".[dev]" +``` + +### Running Commands + +With uv, you can run commands directly without a prior "install" (pg is the Pipeline Generator command): + +```bash +uv run pg --help +uv run pg list +uv run pg gen MONAI/spleen_ct_segmentation --output ./app +``` + +## Usage + +### Complete Workflow Example + +```bash +# 1. List available models +uv run pg list + +# 2. Generate an application from a model +uv run pg gen MONAI/spleen_ct_segmentation --output my_app + +# 3. Run the application +uv run pg run my_app --input /path/to/test/data --output ./results +``` + +### List Available Models + +List all models from configured endpoints: + +```bash +uv run pg list +``` + +Show only MONAI Bundles: + +```bash +uv run pg list --bundles-only +``` + +Show only tested models: + +```bash +uv run pg list --tested-only +``` + +Combine filters: + +```bash +uv run pg list --bundles-only --tested-only # Show only tested MONAI Bundles +``` + +Use different output formats: + +```bash +uv run pg list --format simple # Simple list format +uv run pg list --format json # JSON output +uv run pg list --format table # Default table format +``` + +Use a custom configuration file: + +```bash +uv run pg --config /path/to/config.yaml list +``` + +### Generate MONAI Deploy Application + +Generate an application from a HuggingFace model. Models are specified using the format `organization/model_name` (e.g., `MONAI/spleen_ct_segmentation`): + +```bash +uv run pg gen MONAI/spleen_ct_segmentation --output my_app +``` + +Options: + +- `--output, -o`: Output directory for generated app (default: ./output) +- `--app-name, -n`: Custom application class name (default: derived from model name) +- `--format`: Input/output data format (optional): auto, dicom, or nifti (default: auto) + - For tested models, format is automatically detected from configuration + - For untested models, attempts detection from model metadata +- `--force, -f`: Overwrite existing output directory + +Generate with custom application class name: + +```bash +uv run pg gen MONAI/lung_nodule_ct_detection --output lung_app --app-name LungDetectorApp +``` + +Force overwrite existing directory: + +```bash +uv run pg gen MONAI/spleen_ct_segmentation --output test_app --force +``` + +Override data format (optional - auto-detected for tested models): + +```bash +# Force DICOM format instead of auto-detection +uv run pg gen MONAI/some_model --output my_app --format dicom +``` + +### Run Generated Application + +Run a generated application with automatic environment setup: + +```bash +uv run pg run my_app --input /path/to/input --output /path/to/output +``` + +The `run` command will: + +1. Create a virtual environment if it doesn't exist +1. Install dependencies from requirements.txt +1. Run the application with the specified input/output + +Options: + +- `--input, -i`: Input data directory (required) +- `--output, -o`: Output directory for results (default: ./output) +- `--model, -m`: Override model/bundle path +- `--venv-name`: Virtual environment directory name (default: .venv) +- `--skip-install`: Skip dependency installation +- `--gpu/--no-gpu`: Enable/disable GPU support (default: enabled) + +Examples: + +```bash +# Skip dependency installation (if already installed) +uv run pg run my_app --input test_data --output results --skip-install + +# Run without GPU +uv run pg run my_app --input test_data --output results --no-gpu + +# Use custom model path +uv run pg run my_app --input test_data --output results --model ./custom_model +``` + +## Configuration + +The tool uses a YAML configuration file to define model sources. By default, it looks for `config.yaml` in the package directory. + +Example configuration: + +```yaml +# HuggingFace endpoints to scan for MONAI models +endpoints: + - organization: "MONAI" + base_url: "https://huggingface.co" + description: "Official MONAI organization models" + +# Additional specific models not under the main organization +additional_models: + - model_id: "Project-MONAI/exaonepath" + base_url: "https://huggingface.co" + description: "ExaOnePath model for digital pathology" +``` + +## Generated Application Structure + +When you run `pg gen`, it creates: + +``` +output/ +├── app.py # Main application code +├── app.yaml # Configuration for MONAI Deploy packaging +├── requirements.txt # Python dependencies +├── README.md # Documentation +├── operators/ # Custom operators (if needed) +│ └── nifti_operators.py +└── model/ # Downloaded MONAI Bundle + ├── configs/ + ├── models/ + └── docs/ +``` + +## Development + +### Running Tests + +```bash +# Run all tests +uv run pytest + +# Run with coverage +uv run pytest --cov=pipeline_generator + +# Run specific test file +uv run pytest tests/test_cli.py +``` + +### Code Quality + +```bash +# Format code +uv run black pipeline_generator tests + +# Lint code +uv run flake8 pipeline_generator tests + +# Type checking +uv run mypy pipeline_generator +``` + +## Future Commands + +The CLI is designed to be extensible. Planned commands include: + +- `pg package ` - Package an application using the Holoscan CLI packaging tool + +## License + +This project is part of the MONAI Deploy App SDK and is licensed under the Apache License 2.0. See the main repository's LICENSE file for details. diff --git a/tools/pipeline-generator/docs/design.md b/tools/pipeline-generator/docs/design.md new file mode 100644 index 00000000..eb5ac3a0 --- /dev/null +++ b/tools/pipeline-generator/docs/design.md @@ -0,0 +1,238 @@ +# **MONAI Bundle Integration for MONAI Deploy App SDK & Holoscan SDK** + +## **Objective** + +The goal of this project is to build a robust tool that enables a seamless path for developers to integrate AI models exported in the **MONAI Bundle format** into inference-based applications built with the **MONAI Deploy App SDK** and the **Holoscan SDK**. + +This tool will support: + +- Standard MONAI Bundles (.pt, .ts, .onnx) +- MONAI Bundles exported in **Hugging Face-compatible format** + +By bridging the gap between model packaging and application deployment, this project aims to simplify clinical AI prototyping and deployment across NVIDIA’s edge AI platforms. + +## **Background** + +The **MONAI Bundle** is a standardized format designed to package deep learning models for medical imaging. It includes the model weights, metadata, transforms, and documentation needed to make the model self-contained and portable. + +The **Holoscan SDK** is NVIDIA’s real-time streaming application SDK for AI workloads in healthcare and edge devices. The **MONAI Deploy App SDK** is designed for building composable inference applications, often used in radiology and imaging pipelines. + +As of MONAI Core, bundles can also be exported in a **Hugging Face-compatible format**, which allows sharing through the Hugging Face Model Hub. Supporting this format increases reach and adoption. + +## **Benefits** + +- Speeds up deployment of MONAI-trained models in Holoscan/Deploy pipelines +- Ensures standardized and reproducible model integration +- Makes AI development more accessible to healthcare and edge-AI developers +- Enables the usage of models from Hugging Face directly in clinical-style workflows + +## **Assumptions/Limitations** + +- The tool does not convert input formats given that each model may expect a different type of input +- The tool does not convert output formats given that each model may output a different type of result + +## **Scope** + +This project includes: + +- Support for loading and parsing standard MONAI Bundles (P0) +- Support for Hugging Face-exported MONAI Bundles (P0) +- Integration with MONAI Deploy App SDK (P0) +- Dynamic generation of pre/post-processing pipelines from metadata (P0) +- Integration with Holoscan SDK’s inference operators (P1) +- Tools to validate and prepare MONAI Bundles for deployment (P1) + +## **Key Features** + +- **Bundle Parsing Utility** + - Parses metadata.json, inference.json, and other relevant files + - Extracts model paths, input/output shapes, transform descriptions, and model metadata + - Detects format: .pt, .ts, .onnx, or Hugging Face variant +- **Model Format Support** + - TorchScript (.ts): Loaded with torch.jit.load() + - ONNX (.onnx): Loaded with ONNXRuntime or TensorRT + - PyTorch state dict (.pt): Loaded with model definition code/config + - Hugging Face-compatible: Recognized and unpacked with reference to Hugging Face conventions +- **AI Inference Operator Integration** + - Python and C++ support for TorchScript/ONNX-based inference + - Auto-configures model inputs/outputs based on network_data_format + - Embeds optional postprocessing like argmax, thresholding, etc. +- **Preprocessing/Postprocessing Pipeline** + - Leverages MONAI transforms where applicable + - Builds a dynamic MONAI Deploy pipeline based on the parsed config + - Integrates with existing MONAI Deploy operators + - Builds a dynamic Holoscan Application pipeline based on the parsed config + - Integrates with existing Holoscan operators +- **Pipeline Generation** + - Automatically generate MONAI Deploy App SDK application pipelines from bundle metadata + - Automatically generate Holoscan SDK application pipelines from bundle metadata +- **Tooling** + - Command-line tool to: + - Validate MONAI Bundles + - Convert .pt → .ts/.onnx + - Generate MONAI Deploy and Holoscan-ready configs + - Extract and display metadata (task, inputs, author, etc.) + +## **Pipeline Integration Example** + +Typical MONAI Deploy and Holoscan-based application structure enabled by this module: + +[Source] → [Preprocessing Op] → [Inference Op] → [Postprocessing Op] → [Sink / Visualizer] + +Each operator is configured automatically from the MONAI Bundle metadata, minimizing boilerplate. + +## **Future Directions** + +- Support for multiple models per bundle (e.g. ROI + segmentation) +- Integration with MONAI Label for interactive annotation-driven pipelines +- Hugging Face Model Hub sync/download integration + +## **Tooling** + +This tool will use Python 3.10: + +- A requirements.txt to include all dependencies +- Use poetry for module and dependency management + +## Development Phases + +### Notes + +For each of the following phases, detail describe what is done in the `tools/pipeline-generator/design_phase` directory so you can pickup later, include but not limited to the following: + +- Implementation decisions made +- Code structure and key classes/functions +- Any limitations or assumptions +- Testing approach and results +- Dependencies and versions used +- Ensure no technical debts +- Ensure tests are up-to-date and have good coverage + +### Phase 1 + +First, create a MONAI Deploy application that loads model the spleen_ct_segmentation model from `tools/pipeline-generator/phase_1/spleen_ct_segmentation` (which I downloaded from https://huggingface.co/MONAI/spleen_ct_segmentation/tree/main). The application pipeline shall use pure MONAI Deploy App SDK APIs and operators. + +- The MONAI Deploy application pipeline should include all steps as described above in the *Pipeline Integration Example* section. +- We should parse and implement the preprocessing transforms from the bundle's metadata. +- Ensure configurations are loaded from the [inference.json](tools/pipeline-generator/phase_1/spleen_ct_segmentation/configs/inference.json) file at runtime and not hard coded. +- The input is a directory path; the directory would contain multiple files and the application shall proess all files. +- The output from our application pipeline should be the same as the expected output, same directory structure and data format. We should also compare the application output to the expected output. + +Input (NIfTI): /home/vicchang/Downloads/Task09_Spleen/Task09_Spleen/imagesTs +Model: tools/pipeline-generator/phase_1/spleen_ct_segmentation/models/model.ts +Expected Output: tools/pipeline-generator/phase_1/spleen_ct_segmentation/eval + +Note: we may need to modify the existing [monai_bundle_inference_operator](monai/deploy/operators/monai_bundle_inference_operator.py) to support loading from a directory instead of a ZIP file. We should modify the py file directly and not extend it. Ensure to keep existing ZIP file support. + +Note: refer to [samples](/home/vicchang/sc/github/monai/monai-deploy-app-sdk/examples) for how to compose a MONAI Deploy application. Reuse all operators if possible. For example, if there are Nifti loaders, then do not recreate one. + +`For this phase, assume we use pure MONAI Deploy App SDK end-to-end.` + +### Phase 2 + +Create a CLI with a command to list available models from https://huggingface.co/MONAI. It should pull all available models using HuggingFace Python API at runtime. +However, a YAML config file should have a list of endpoints to scan the models from, we will start with https://huggingface.co/MONAI but later add models listed in section Phase 7. +The CLI should be able to support other commands later. For example, in 0.2, we need to add a command to generate an application and 0.3 to run the generated application. + +```bash +pg list +``` + +Note: this new project solution shall be independent from Phase 1. This project shall use poetry for dependency management and include unit test. + +### Phase 3 + +- Generate a MONAI Deploy-based Pipeline on selected a select MONAI Bundle from https://huggingface.co/MONAI. There are currently 40 models available. The Python module shall output the following: + +1. app.py that include the end-to-end MONAI Deploy pipeline as outlined in the "Pipeline Integration Example" section above. +1. app.yaml with all configurations +1. Any models files and configurations from the downloaded model +1. READMD.md with instructions on how to run the app and info about the selected model. + +Important: download all files from the model repository. +Note: there are reference applications in [examples](/home/vicchang/sc/github/monai/monai-deploy-app-sdk/examples). + +A sample directory structure shall look like: + +```bash +root/ +├── app.py +├── app.yaml +└── model/ + └── (model files downloaded from HuggingFace repository) +``` + +Implement the solution with ability to generate a MONAI Deploy application based on the selected model. + +- Jinja2 for main code templates - Perfect for generating app.py with variable operator configurations +- Pydantic/dataclasses for configuration models - To validate and structure app.yaml data +- YAML library for configuration generation - Direct YAML output from Python objects +- Poetry for project management (as specified in your design) + +```bash +pg gen spleen_ct_segmentation --ouput [path-to-generated-output] #for during testing we should always use ./output to store generated applications +``` + +### Phase 4 + +Add a new CLI command to run the newly generated app with the application directory, test data and output directory as arguments. +It should create a virtual environment, install dependencies and run the application. + +```bash +pg run path-to-generated-app --input test-data-dir --output result-dir +``` + +### Phase 5 + +Replace poetry with uv. + +- Ensure all existing docs are updated +- Ensure all existing commands still work +- Run unit test and ensure coverage is at least 90% + +### Phase 6 + +Add support for MONAI/Llama3-VILA-M3-3B model. + +- Create new operators for the model in 'monai/deploy/operators' so it can be reused by other Llama3 models. The first operator should be able to take a directory as input and scan for a prompts.yaml file in the following format: + +```yaml +defaults: + max_new_tokens: 256 + temperature: 0.2 + top_p: 0.9 +prompts: + - prompt: Summarize key findings. + image: img1.png + output: json + - prompt: Is there a focal lesion? Answer yes/no and describe location. + image: img2.png + output: image + max_new_tokens: 128 +``` + +Where `prompts.prompt` is the prompt fora set of images and `prompts.image` is an image associated with the prompt. The `prompts.output` indicates the type to expect for each prompt, it could be one of the following: json (see below for sample), image (generate a new image in the output directory with the AI response), image_overlay (this could be segmentation mask, bounding boxes etc...). + +The first operator (VLMPromptsLoaderOperator) shall have a single output port that includes image + prompt + output_type + request_id (filename + datetime) and shall emit one prompt only each time compute is called. The operator shall end the application once all prompts have been processed (see monai/deploy/operators/image_directory_loader_operator.py L92-96). + +The second operator (Llama3VILAInferenceOperator) takes the input from first operator and run the model. Once the model is ready with results, output it to the output port for the last operator. + +The third and last operator (VLMResultsWriterOperator) shall take input from the first operator and the results from second operator and then write the results to the results directory specified by the user. The type of data to write to disk depends on the output type defined in the prompt. + +The output of the JSON should be in the following format: + +```json +{ + "prompt": "original prompt", + "response": "AI generated response" +} +``` + +Update config.yaml with the new model. + +Note: no changes to the pg run command. +Note: in this phase, we will support a single 2D image (PNG/JPEG) only. +Note: Since this model, the prompts.yaml, supports custom input/output formats, we will use "custom" as the input_type and output_type in the [config.yaml](tools/pipeline-generator/pipeline_generator/config/config.yaml). +Note: results are saved to the destination directory from pg run --output parameter. + +**Phase 6 Status**: ✅ Completed - All three operators created and added to MONAI Deploy. The model appears in the pipeline generator list. Template integration requires additional work for full "custom" type support. diff --git a/tools/pipeline-generator/pipeline_generator/__init__.py b/tools/pipeline-generator/pipeline_generator/__init__.py new file mode 100644 index 00000000..8558a6a1 --- /dev/null +++ b/tools/pipeline-generator/pipeline_generator/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2025 MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Pipeline Generator for MONAI Deploy and Holoscan applications.""" + +__version__ = "0.1.0" diff --git a/tools/pipeline-generator/pipeline_generator/cli/__init__.py b/tools/pipeline-generator/pipeline_generator/cli/__init__.py new file mode 100644 index 00000000..761d6a13 --- /dev/null +++ b/tools/pipeline-generator/pipeline_generator/cli/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2025 MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""CLI module for Pipeline Generator.""" + +from .main import cli + +__all__ = ["cli"] diff --git a/tools/pipeline-generator/pipeline_generator/cli/main.py b/tools/pipeline-generator/pipeline_generator/cli/main.py new file mode 100644 index 00000000..085967c7 --- /dev/null +++ b/tools/pipeline-generator/pipeline_generator/cli/main.py @@ -0,0 +1,307 @@ +# Copyright 2025 MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Main CLI entry point for Pipeline Generator.""" + +import logging +from pathlib import Path +from typing import List, Optional, Set + +import click +from rich.console import Console +from rich.logging import RichHandler +from rich.table import Table + +from ..config import load_config +from ..core import HuggingFaceClient, ModelInfo +from ..generator import AppGenerator +from .run import run as run_command + +# Set up logging with Rich +logging.basicConfig( + level=logging.INFO, + format="%(message)s", + handlers=[RichHandler(show_time=False, show_path=False)], +) +logger = logging.getLogger(__name__) + +console = Console() + + +@click.group() +@click.version_option() +@click.option("--config", "-c", type=click.Path(exists=True), help="Path to configuration file") +@click.pass_context +def cli(ctx: click.Context, config: Optional[str]) -> None: + """Pipeline Generator - Generate MONAI Deploy and Holoscan pipelines from MONAI Bundles.""" + # Store config in context for subcommands + ctx.ensure_object(dict) + config_path = Path(config) if config else None + ctx.obj["config_path"] = config_path + + # Load settings + from ..config.settings import load_config + + settings = load_config(config_path) + ctx.obj["settings"] = settings + + +@cli.command() +@click.option( + "--format", + "-f", + type=click.Choice(["table", "simple", "json"]), + default="table", + help="Output format", +) +@click.option("--bundles-only", "-b", is_flag=True, help="Show only MONAI Bundles") +@click.option("--tested-only", "-t", is_flag=True, help="Show only tested models") +@click.pass_context +def list(ctx: click.Context, format: str, bundles_only: bool, tested_only: bool) -> None: + """List available models from configured endpoints. + + Args: + ctx: Click context containing configuration + format: Output format (table, simple, or json) + bundles_only: If True, show only MONAI Bundles + tested_only: If True, show only tested models + + Example: + pg list --format table --bundles-only + """ + + # Load configuration + config_path = ctx.obj.get("config_path") + settings = load_config(config_path) + + # Get set of tested model IDs from configuration + tested_models = set() + for endpoint in settings.endpoints: + for model in endpoint.models: + tested_models.add(model.model_id) + + # Create HuggingFace client + client = HuggingFaceClient() + + # Fetch models from all endpoints + console.print("[blue]Fetching models from HuggingFace...[/blue]") + models = client.list_models_from_endpoints(settings.get_all_endpoints()) + + # Filter for bundles if requested + if bundles_only: + models = [m for m in models if m.is_monai_bundle] + + # Filter for tested models if requested + if tested_only: + models = [m for m in models if m.model_id in tested_models] + + # Sort models by name + models.sort(key=lambda m: m.display_name) + + # Display results based on format + if format == "table": + _display_table(models, tested_models) + elif format == "simple": + _display_simple(models, tested_models) + elif format == "json": + _display_json(models, tested_models) + + # Summary + bundle_count = sum(1 for m in models if m.is_monai_bundle) + tested_count = sum(1 for m in models if m.model_id in tested_models) + console.print( + f"\n[green]Total models: {len(models)} (MONAI Bundles: {bundle_count}, Verified: {tested_count})[/green]" + ) + + +@cli.command() +@click.argument("model_id") +@click.option( + "--output", + "-o", + type=click.Path(), + default="./output", + help="Output directory for generated app", +) +@click.option("--app-name", "-n", help="Custom application class name") +@click.option( + "--format", + type=click.Choice(["auto", "dicom", "nifti"]), + default="auto", + help="Input/output format (optional): auto (uses config for tested models), dicom, or nifti", +) +@click.option("--force", "-f", is_flag=True, help="Overwrite existing output directory") +@click.pass_context +def gen( + ctx: click.Context, + model_id: str, + output: str, + app_name: Optional[str], + format: str, + force: bool, +) -> None: + """Generate a MONAI Deploy application from a HuggingFace model. + + Downloads the specified model from HuggingFace and generates a complete + MONAI Deploy application including app.py, app.yaml, requirements.txt, + README.md, and the model files. + + Args: + model_id: HuggingFace model ID (e.g., 'MONAI/spleen_ct_segmentation') + output: Output directory for generated app (default: ./output) + app_name: Custom application class name (default: derived from model) + format: Input/output format - 'auto' (detect), 'dicom', or 'nifti' + force: Overwrite existing output directory if True + + Example: + pg gen MONAI/spleen_ct_segmentation --output my_app + + Raises: + click.Abort: If output directory exists and force is False + """ + output_path = Path(output) + + # Check if output directory exists + if output_path.exists() and not force: + if any(output_path.iterdir()): # Directory is not empty + console.print(f"[red]Error: Output directory '{output_path!r}' already exists and is not empty.[/red]") + console.print("Use --force to overwrite or choose a different output directory.") + raise click.Abort() + + # Create generator with settings from context + settings = ctx.obj.get("settings") if ctx.obj else None + generator = AppGenerator(settings=settings) + + console.print(f"[blue]Generating MONAI Deploy application for model: {model_id}[/blue]") + console.print(f"[blue]Output directory: {output_path}[/blue]") + console.print(f"[blue]Format: {format}[/blue]") + + try: + # Generate the application + app_path = generator.generate_app( + model_id=model_id, + output_dir=output_path, + app_name=app_name, + data_format=format, + ) + + console.print("\n[green]✓ Application generated successfully![/green]") + console.print("\n[bold]Generated files:[/bold]") + + # List generated files + for file in output_path.rglob("*"): + if file.is_file(): + relative_path = file.relative_to(output_path) + console.print(f" • {relative_path}") + + console.print("\n[bold]Next steps:[/bold]") + console.print("\n[green]Option 1: Run with uv (recommended)[/green]") + console.print(f" [cyan]uv run pg run {output_path} --input /path/to/input --output /path/to/output[/cyan]") + console.print("\n[green]Option 2: Run with pg directly[/green]") + console.print(f" [cyan]pg run {output_path} --input /path/to/input --output /path/to/output[/cyan]") + console.print("\n[dim]Option 3: Run manually[/dim]") + console.print(" 1. Navigate to the application directory:") + console.print(f" [cyan]cd {output_path}[/cyan]") + console.print(" 2. (Optional) Create and activate virtual environment:") + console.print(" [cyan]python -m venv venv[/cyan]") + console.print(" [cyan]source venv/bin/activate # Linux/Mac[/cyan]") + console.print(" [cyan]# or: venv\\Scripts\\activate # Windows[/cyan]") + console.print(" 3. Install dependencies:") + console.print(" [cyan]pip install -r requirements.txt[/cyan]") + console.print(" 4. Run the application:") + console.print(" [cyan]python app.py -i /path/to/input -o /path/to/output[/cyan]") + + except Exception as e: + console.print(f"[red]Error generating application: {e}[/red]") + logger.exception("Generation failed") + raise click.Abort() from e + + +def _display_table(models: List[ModelInfo], tested_models: Set[str]) -> None: + """Display models in a rich table format. + + Args: + models: List of ModelInfo objects to display + tested_models: Set of tested model IDs + """ + table = Table(title="Available Models", show_header=True, header_style="bold magenta") + table.add_column("Model ID", style="cyan", width=40) + table.add_column("Name", style="white") + table.add_column("Type", style="green") + table.add_column("Status", style="blue", width=10) + table.add_column("Downloads", justify="right", style="yellow") + table.add_column("Likes", justify="right", style="red") + + for model in models: + model_type = "[green]MONAI Bundle[/green]" if model.is_monai_bundle else "Model" + status = "[bold green]✓ Verified[/bold green]" if model.model_id in tested_models else "" + table.add_row( + model.model_id, + model.display_name, + model_type, + status, + str(model.downloads or "N/A"), + str(model.likes or "N/A"), + ) + + console.print(table) + + +def _display_simple(models: List[ModelInfo], tested_models: Set[str]) -> None: + """Display models in a simple list format. + + Shows each model with emoji indicators: + - 📦 for MONAI Bundle, 📄 for regular model + - ✓ for tested models + + Args: + models: List of ModelInfo objects to display + tested_models: Set of tested model IDs + """ + for model in models: + bundle_marker = "📦" if model.is_monai_bundle else "📄" + tested_marker = " ✓" if model.model_id in tested_models else "" + console.print(f"{bundle_marker} {model.model_id} - {model.display_name}{tested_marker}") + + +def _display_json(models: List[ModelInfo], tested_models: Set[str]) -> None: + """Display models in JSON format. + + Outputs a JSON array of model information suitable for programmatic consumption. + + Args: + models: List of ModelInfo objects to display + tested_models: Set of tested model IDs + """ + import json + + data = [ + { + "model_id": m.model_id, + "name": m.display_name, + "is_monai_bundle": m.is_monai_bundle, + "is_tested": m.model_id in tested_models, + "downloads": m.downloads, + "likes": m.likes, + "tags": m.tags, + } + for m in models + ] + + console.print_json(json.dumps(data, indent=2)) + + +# Add the run command to CLI +cli.add_command(run_command) + + +if __name__ == "__main__": + cli() diff --git a/tools/pipeline-generator/pipeline_generator/cli/run.py b/tools/pipeline-generator/pipeline_generator/cli/run.py new file mode 100644 index 00000000..44711214 --- /dev/null +++ b/tools/pipeline-generator/pipeline_generator/cli/run.py @@ -0,0 +1,364 @@ +# Copyright 2025 MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Run command for executing generated MONAI Deploy applications.""" + +import logging +import os +import subprocess +import sys +from pathlib import Path +from typing import Optional + +import click +from rich.console import Console +from rich.progress import Progress, SpinnerColumn, TextColumn + +logger = logging.getLogger(__name__) +console = Console() + + +def _validate_results(output_dir: Path) -> tuple[bool, str]: + """Validate that the application actually generated results. + + Args: + output_dir: Path to the output directory + + Returns: + Tuple of (success, message) where success is True if validation passed + """ + if not output_dir.exists(): + return False, f"Output directory does not exist: {output_dir}" + + # Check if any files were generated in the output directory + output_files = list(output_dir.rglob("*")) + result_files = [f for f in output_files if f.is_file()] + + if not result_files: + return False, f"No result files generated in {output_dir}" + + # Count different types of output files + json_files = [f for f in result_files if f.suffix.lower() == ".json"] + nifti_files = [f for f in result_files if f.suffix.lower() in [".nii", ".gz"]] + image_files = [f for f in result_files if f.suffix.lower() in [".png", ".jpg", ".jpeg", ".tiff"]] + other_files = [f for f in result_files if f not in json_files + nifti_files + image_files] + + file_summary = [] + if json_files: + file_summary.append(f"{len(json_files)} JSON files") + if nifti_files: + file_summary.append(f"{len(nifti_files)} NIfTI files") + if image_files: + file_summary.append(f"{len(image_files)} image files") + if other_files: + file_summary.append(f"{len(other_files)} other files") + + summary = ", ".join(file_summary) if file_summary else f"{len(result_files)} files" + return True, f"Generated {summary}" + + +@click.command() +@click.argument( + "app_path", + type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path), +) +@click.option( + "--input", + "-i", + "input_dir", + required=True, + type=click.Path(exists=True, path_type=Path), + help="Input data directory", +) +@click.option( + "--output", + "-o", + "output_dir", + type=click.Path(path_type=Path), + default="./output", + help="Output directory for results", +) +@click.option( + "--model", + "-m", + "model_path", + type=click.Path(exists=True, path_type=Path), + help="Override model/bundle path", +) +@click.option("--venv-name", default=".venv", help="Virtual environment directory name") +@click.option("--skip-install", is_flag=True, help="Skip dependency installation") +@click.option("--gpu/--no-gpu", default=True, help="Enable/disable GPU support") +def run( + app_path: str, + input_dir: str, + output_dir: str, + model_path: Optional[str], + venv_name: str, + skip_install: bool, + gpu: bool, +) -> None: + """Run a generated MONAI Deploy application. + + This command automates the process of setting up and running a MONAI Deploy + application by managing virtual environments, dependencies, and execution. + + Platform Requirements: + Linux/Unix operating systems only (consistent with MONAI Deploy App SDK) + + Steps performed: + 1. Create a virtual environment if it doesn't exist + 2. Install dependencies from requirements.txt (unless --skip-install) + 3. Run the application with the specified input/output directories + + Args: + app_path: Path to the generated application directory + input_dir: Directory containing input data (DICOM or NIfTI files) + output_dir: Directory where results will be saved + model_path: Optional override for model/bundle path + venv_name: Name of virtual environment directory (default: .venv) + skip_install: Skip dependency installation if True + gpu: Enable GPU support (default: True) + + Example: + pg run ./my_app --input ./test_data --output ./results --no-gpu + + Raises: + click.Abort: If app.py or requirements.txt not found, or if execution fails + """ + app_path_obj = Path(app_path).resolve() + input_dir_obj = Path(input_dir).resolve() + output_dir_obj = Path(output_dir).resolve() + + # Check if app.py exists + app_file = app_path_obj / "app.py" + if not app_file.exists(): + console.print(f"[red]Error: app.py not found in {app_path}[/red]") + raise click.Abort() + + # Check requirements.txt + requirements_file = app_path_obj / "requirements.txt" + if not requirements_file.exists(): + console.print(f"[red]Error: requirements.txt not found in {app_path}[/red]") + raise click.Abort() + + venv_path = app_path_obj / venv_name + + console.print(f"[blue]Running MONAI Deploy application from: {app_path_obj}[/blue]") + console.print(f"[blue]Input: {input_dir_obj}[/blue]") + console.print(f"[blue]Output: {output_dir_obj}[/blue]") + + # Create output directory if it doesn't exist + output_dir_obj.mkdir(parents=True, exist_ok=True) + + # Step 1: Create virtual environment if needed + if not venv_path.exists(): + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + console=console, + ) as progress: + task = progress.add_task("Creating virtual environment...", total=None) + try: + subprocess.run( + [sys.executable, "-m", "venv", str(venv_path)], + check=True, + capture_output=True, + text=True, + ) + progress.update(task, description="[green]Virtual environment created") + except subprocess.CalledProcessError as e: + console.print(f"[red]Error creating virtual environment: {e.stderr}[/red]") + raise click.Abort() from e + else: + console.print(f"[dim]Using existing virtual environment: {venv_name}[/dim]") + + # Determine python executable in venv (Linux/Unix only) + python_exe = venv_path / "bin" / "python" + pip_exe = venv_path / "bin" / "pip" + + # Step 2: Install dependencies + if not skip_install: + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + console=console, + ) as progress: + task = progress.add_task("Installing dependencies...", total=None) + + # Ensure pip/setuptools/wheel are up to date for Python 3.12+ + try: + # Ensure pip is present and upgraded inside the venv + subprocess.run( + [str(python_exe), "-m", "ensurepip", "--upgrade"], + check=False, + capture_output=True, + text=True, + ) + subprocess.run( + [ + str(pip_exe), + "install", + "--upgrade", + "pip", + "setuptools", + "wheel", + ], + check=True, + capture_output=True, + text=True, + ) + except subprocess.CalledProcessError as e: + console.print( + ( + "[yellow]Warning: Failed to upgrade pip/setuptools/wheel: " + f"{e.stderr}\nContinuing with dependency installation...[/yellow]" + ) + ) + + # Detect local SDK checkout and install editable to expose local operators + local_sdk_installed = False + script_path = Path(__file__).resolve() + sdk_path = script_path.parent.parent.parent.parent.parent + if (sdk_path / "monai" / "deploy").exists() and (sdk_path / "setup.py").exists(): + console.print(f"[dim]Found local SDK at: {sdk_path}[/dim]") + + # Install local SDK first + try: + subprocess.run( + [str(pip_exe), "install", "-e", str(sdk_path)], + check=True, + capture_output=True, + text=True, + ) + local_sdk_installed = True + except subprocess.CalledProcessError as e: + console.print(f"[yellow]Warning: Failed to install local SDK: {e.stderr}[/yellow]") + + # Install requirements + try: + req_path_to_use = requirements_file + temp_req_path = None + + if local_sdk_installed: + # Filter out SDK line to avoid overriding local editable install + try: + raw = requirements_file.read_text() + filtered_lines = [] + for line in raw.splitlines(): + s = line.strip() + if not s or s.startswith("#"): + filtered_lines.append(line) + continue + if s.lower().startswith("monai-deploy-app-sdk"): + continue + filtered_lines.append(line) + temp_req_path = app_path_obj / ".requirements.filtered.txt" + temp_req_path.write_text("\n".join(filtered_lines) + "\n") + req_path_to_use = temp_req_path + console.print("[dim]Using filtered requirements without monai-deploy-app-sdk[/dim]") + except Exception as fr: + console.print( + f"[yellow]Warning: Failed to filter requirements: {fr}. Proceeding with original requirements.[/yellow]" + ) + req_path_to_use = requirements_file + + subprocess.run( + [str(pip_exe), "install", "-r", str(req_path_to_use), "-q"], + check=True, + capture_output=True, + text=True, + ) + + # Re-assert local editable SDK in case it was overridden + if local_sdk_installed: + try: + subprocess.run( + [str(pip_exe), "install", "-e", str(sdk_path)], + check=True, + capture_output=True, + text=True, + ) + except subprocess.CalledProcessError as re: + console.print(f"[yellow]Warning: Re-installing local SDK failed: {re.stderr}[/yellow]") + + progress.update(task, description="[green]Dependencies installed") + except subprocess.CalledProcessError as e: + console.print(f"[red]Error installing dependencies: {e.stderr}[/red]") + raise click.Abort() from e + + # Step 3: Run the application + console.print("\n[green]Starting application...[/green]\n") + + # Build command + cmd = [ + str(python_exe), + str(app_file), + "-i", + str(input_dir_obj), + "-o", + str(output_dir_obj), + ] + + # Add model path if provided + if model_path: + cmd.extend(["-m", str(model_path)]) + + # Set environment variables + env = os.environ.copy() + if not gpu: + env["CUDA_VISIBLE_DEVICES"] = "" + + try: + # Run the application + process = subprocess.Popen( + cmd, + cwd=str(app_path_obj), + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + ) + + # Stream output in real-time + if process.stdout: + for line in process.stdout: + print(line, end="") + + # Wait for completion + return_code = process.wait() + + if return_code == 0: + # Validate that results were actually generated + success, message = _validate_results(output_dir_obj) + if success: + console.print("\n[green]✓ Application completed successfully![/green]") + console.print(f"[green]Results saved to: {output_dir_obj}[/green]") + console.print(f"[dim]{message}[/dim]") + else: + console.print(f"\n[red]✗ Application completed but failed validation: {message}[/red]") + console.print("[red]This usually indicates operator connection issues or processing failures.[/red]") + raise click.Abort() from None + else: + console.print(f"\n[red]✗ Application failed with exit code: {return_code}[/red]") + raise click.Abort() from None + + except KeyboardInterrupt as e: + console.print("\n[yellow]Application interrupted by user[/yellow]") + process.terminate() + raise click.Abort() from e + except Exception as e: + console.print(f"[red]Error running application: {e}[/red]") + raise click.Abort() from e + + +if __name__ == "__main__": + run() diff --git a/tools/pipeline-generator/pipeline_generator/config/__init__.py b/tools/pipeline-generator/pipeline_generator/config/__init__.py new file mode 100644 index 00000000..3c20c3b8 --- /dev/null +++ b/tools/pipeline-generator/pipeline_generator/config/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2025 MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Configuration module for Pipeline Generator.""" + +from .settings import Endpoint, Settings, load_config + +__all__ = ["Settings", "Endpoint", "load_config"] diff --git a/tools/pipeline-generator/pipeline_generator/config/config.yaml b/tools/pipeline-generator/pipeline_generator/config/config.yaml new file mode 100644 index 00000000..2749d84c --- /dev/null +++ b/tools/pipeline-generator/pipeline_generator/config/config.yaml @@ -0,0 +1,93 @@ +# Copyright 2025 MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Pipeline Generator Configuration + +# HuggingFace endpoints to scan for MONAI models +endpoints: + - organization: "MONAI" + base_url: "https://huggingface.co" + description: "Official MONAI organization models" + models: # tested models + - model_id: "MONAI/spleen_ct_segmentation" + input_type: "nifti" + output_type: "nifti" + - model_id: "MONAI/multi_organ_segmentation" + input_type: "nifti" + output_type: "nifti" + - model_id: "MONAI/breast_density_classification" + input_type: "image" + output_type: "json" + - model_id: "MONAI/endoscopic_tool_segmentation" + input_type: "image" + output_type: "image_overlay" + configs: + - channel_first: false + - model_id: "MONAI/wholeBrainSeg_Large_UNEST_segmentation" + input_type: "nifti" + output_type: "nifti" + - model_id: "MONAI/wholeBody_ct_segmentation" + input_type: "nifti" + output_type: "nifti" + - model_id: "MONAI/swin_unetr_btcv_segmentation" + input_type: "nifti" + output_type: "nifti" + - model_id: "MONAI/pancreas_ct_dints_segmentation" + input_type: "nifti" + output_type: "nifti" + - model_id: "MONAI/pediatric_abdominal_ct_segmentation" + input_type: "nifti" + output_type: "nifti" + dependencies: + - nibabel>=3.2.0 # Required for NIfTI file I/O support + - itk>=5.3.0 # Required for ITK-based image readers/writers + - model_id: "MONAI/Llama3-VILA-M3-3B" + input_type: "custom" + output_type: "custom" + dependencies: + - transformers>=4.44.0 + - torch>=2.0.0 + - Pillow>=8.0.0 + - PyYAML>=6.0 + - model_id: "MONAI/Llama3-VILA-M3-8B" + input_type: "custom" + output_type: "custom" + dependencies: + - transformers>=4.44.0 + - torch>=2.0.0 + - Pillow>=8.0.0 + - PyYAML>=6.0 + - model_id: "MONAI/Llama3-VILA-M3-13B" + input_type: "custom" + output_type: "custom" + dependencies: + - transformers>=4.44.0 + - torch>=2.0.0 + - Pillow>=8.0.0 + - PyYAML>=6.0 + - model_id: "MONAI/example_spleen_segmentation" + input_type: "nifti" + output_type: "nifti" + dependencies: + - torch>=1.11.0,<3.0.0 + - numpy>=1.21.2,<2.0.0 + - monai>=1.3.0 + - nibabel>=3.0.0 + +additional_models: + - model_id: "LGAI-EXAONE/EXAONEPath" + base_url: "https://huggingface.co" + description: "ExaOnePath - Pathology foundation model with stain normalization" + model_type: "pathology" + - model_id: "LGAI-EXAONE/EXAONEPath-CRC-MSI-Predictor" + base_url: "https://huggingface.co" + description: "ExaOnePath CRC MSI Predictor - Colorectal cancer microsatellite instability prediction" + model_type: "pathology" diff --git a/tools/pipeline-generator/pipeline_generator/config/settings.py b/tools/pipeline-generator/pipeline_generator/config/settings.py new file mode 100644 index 00000000..8ae2ab60 --- /dev/null +++ b/tools/pipeline-generator/pipeline_generator/config/settings.py @@ -0,0 +1,136 @@ +# Copyright 2025 MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Settings and configuration management for Pipeline Generator.""" + +from pathlib import Path +from typing import Any, Dict, List, Optional, Union + +import yaml # type: ignore +from pydantic import BaseModel, Field + + +class ModelConfig(BaseModel): + """Configuration for a specific model.""" + + model_id: str = Field(..., description="Model ID (e.g., 'MONAI/spleen_ct_segmentation')") + input_type: str = Field("nifti", description="Input data type: 'nifti', 'dicom', 'image'") + output_type: str = Field( + "nifti", + description="Output data type: 'nifti', 'dicom', 'json', 'image_overlay'", + ) + configs: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = Field( + None, + description="Additional template configs per model (dict or list of dicts)", + ) + dependencies: Optional[List[str]] = Field( + default=[], + description="Additional pip requirement specifiers to include in generated requirements.txt", + ) + + +class Endpoint(BaseModel): + """Model endpoint configuration.""" + + organization: Optional[str] = Field(None, description="HuggingFace organization name") + model_id: Optional[str] = Field(None, description="Specific model ID") + base_url: str = Field("https://huggingface.co", description="Base URL for the endpoint") + description: str = Field("", description="Endpoint description") + model_type: Optional[str] = Field( + None, + description="Model type: segmentation, pathology, multimodal, multimodal_llm", + ) + models: List[ModelConfig] = Field(default_factory=list, description="Tested models with known data types") + + +class Settings(BaseModel): + """Application settings.""" + + endpoints: List[Endpoint] = Field(default_factory=list) + additional_models: List[Endpoint] = Field(default_factory=list) + + @classmethod + def from_yaml(cls, path: Path) -> "Settings": + """Load settings from YAML file. + + Args: + path: Path to YAML configuration file + + Returns: + Settings object initialized from YAML data + """ + with open(path, "r") as f: + data = yaml.safe_load(f) + return cls(**data) + + def get_all_endpoints(self) -> List[Endpoint]: + """Get all endpoints including additional models. + + Combines the main endpoints list with additional_models to provide + a single list of all configured endpoints. + + Returns: + List of all Endpoint configurations + """ + return self.endpoints + self.additional_models + + def get_model_config(self, model_id: str) -> Optional[ModelConfig]: + """Get model configuration for a specific model ID. + + Searches through all endpoints' model configurations to find + the configuration for the specified model ID. + + Args: + model_id: The model ID to search for + + Returns: + ModelConfig if found, None otherwise + """ + for endpoint in self.get_all_endpoints(): + for model in endpoint.models: + if model.model_id == model_id: + return model + return None + + +def load_config(config_path: Optional[Path] = None) -> Settings: + """Load configuration from file or use defaults. + + Attempts to load configuration from the specified path, falling back to + a config.yaml in the package directory, or finally to default settings + if no config file is found. + + Args: + config_path: Optional path to configuration file + + Returns: + Settings object with loaded or default configuration + """ + if config_path is None: + # Try to find config.yaml in package + package_dir = Path(__file__).parent + config_path = package_dir / "config.yaml" + + if config_path.exists(): + return Settings.from_yaml(config_path) + + # Return default settings if no config file found + return Settings( + endpoints=[ + Endpoint( + organization="MONAI", + model_id=None, + base_url="https://huggingface.co", + description="Official MONAI organization models", + model_type=None, + ) + ] + ) diff --git a/tools/pipeline-generator/pipeline_generator/core/__init__.py b/tools/pipeline-generator/pipeline_generator/core/__init__.py new file mode 100644 index 00000000..2041478a --- /dev/null +++ b/tools/pipeline-generator/pipeline_generator/core/__init__.py @@ -0,0 +1,17 @@ +# Copyright 2025 MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Core functionality for Pipeline Generator.""" + +from .hub_client import HuggingFaceClient +from .models import ModelInfo + +__all__ = ["ModelInfo", "HuggingFaceClient"] diff --git a/tools/pipeline-generator/pipeline_generator/core/hub_client.py b/tools/pipeline-generator/pipeline_generator/core/hub_client.py new file mode 100644 index 00000000..cddbc6a8 --- /dev/null +++ b/tools/pipeline-generator/pipeline_generator/core/hub_client.py @@ -0,0 +1,148 @@ +# Copyright 2025 MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""HuggingFace Hub client for fetching model information.""" + +import logging +from typing import Any, List, Optional + +from huggingface_hub import HfApi, list_models, model_info +from huggingface_hub.utils import HfHubHTTPError + +from ..config import Endpoint +from .models import ModelInfo + +logger = logging.getLogger(__name__) + + +class HuggingFaceClient: + """Client for interacting with HuggingFace Hub.""" + + def __init__(self) -> None: + """Initialize the HuggingFace Hub client.""" + self.api = HfApi() + + def list_models_from_organization(self, organization: str) -> List[ModelInfo]: + """List all models from a HuggingFace organization. + + Args: + organization: HuggingFace organization name (e.g., 'MONAI') + + Returns: + List of ModelInfo objects + """ + models = [] + + try: + # Use the HuggingFace API to list models + for model in list_models(author=organization): + model_data = self._extract_model_info(model) + models.append(model_data) + + except Exception as e: + logger.error(f"Error listing models from {organization}: {e}") + + return models + + def get_model_info(self, model_id: str) -> Optional[ModelInfo]: + """Get detailed information about a specific model. + + Args: + model_id: Model ID (e.g., 'MONAI/spleen_ct_segmentation') + + Returns: + ModelInfo object or None if not found + """ + try: + model = model_info(model_id) + return self._extract_model_info(model) + except HfHubHTTPError as e: + logger.error(f"Model {model_id} not found: {e}") + return None + except Exception as e: + logger.error(f"Error fetching model {model_id}: {e}") + return None + + def list_models_from_endpoints(self, endpoints: List[Endpoint]) -> List[ModelInfo]: + """List models from all configured endpoints. + + Args: + endpoints: List of endpoint configurations + + Returns: + List of ModelInfo objects from all endpoints + """ + all_models = [] + + for endpoint in endpoints: + if endpoint.organization: + # List all models from organization + logger.info(f"Fetching models from organization: {endpoint.organization}") + models = self.list_models_from_organization(endpoint.organization) + all_models.extend(models) + + elif endpoint.model_id: + # Get specific model + logger.info(f"Fetching model: {endpoint.model_id}") + model = self.get_model_info(endpoint.model_id) + if model: + all_models.append(model) + + return all_models + + def _extract_model_info(self, model_data: Any) -> ModelInfo: + """Extract ModelInfo from HuggingFace model data. + + Args: + model_data: Model data from HuggingFace API + + Returns: + ModelInfo object + """ + # Check if this is a MONAI Bundle + is_monai_bundle = False + bundle_metadata = None + + # Check tags for MONAI-related tags + tags = getattr(model_data, "tags", []) + if any("monai" in tag.lower() for tag in tags): + is_monai_bundle = True + + # Check if metadata.json exists in the model files + try: + if hasattr(model_data, "siblings"): + file_names = [f.rfilename for f in model_data.siblings] + if any("metadata.json" in f for f in file_names): + is_monai_bundle = True + except Exception: + pass + + # Extract description from cardData if available + description = None + card_data = getattr(model_data, "cardData", None) + if card_data and isinstance(card_data, dict): + description = card_data.get("description") + if not description: + description = getattr(model_data, "description", None) + + return ModelInfo( + model_id=model_data.modelId, + name=getattr(model_data, "name", model_data.modelId), + author=getattr(model_data, "author", None), + description=description, + downloads=getattr(model_data, "downloads", None), + likes=getattr(model_data, "likes", None), + created_at=getattr(model_data, "created_at", None), + updated_at=getattr(model_data, "lastModified", None), + tags=tags, + is_monai_bundle=is_monai_bundle, + bundle_metadata=bundle_metadata, + ) diff --git a/tools/pipeline-generator/pipeline_generator/core/models.py b/tools/pipeline-generator/pipeline_generator/core/models.py new file mode 100644 index 00000000..f43269f2 --- /dev/null +++ b/tools/pipeline-generator/pipeline_generator/core/models.py @@ -0,0 +1,60 @@ +# Copyright 2025 MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Data models for Pipeline Generator.""" + +from datetime import datetime +from typing import Any, Dict, List, Optional + +from pydantic import BaseModel, Field + + +class ModelInfo(BaseModel): + """Model information from HuggingFace.""" + + model_id: str = Field(..., description="Model ID (e.g., 'MONAI/spleen_ct_segmentation')") + name: str = Field(..., description="Model name") + author: Optional[str] = Field(None, description="Model author/organization") + description: Optional[str] = Field(None, description="Model description") + downloads: Optional[int] = Field(None, description="Number of downloads") + likes: Optional[int] = Field(None, description="Number of likes") + created_at: Optional[datetime] = Field(None, description="Creation date") + updated_at: Optional[datetime] = Field(None, description="Last update date") + tags: List[str] = Field(default_factory=list, description="Model tags") + is_monai_bundle: bool = Field(False, description="Whether this is a MONAI Bundle") + bundle_metadata: Optional[Dict[str, Any]] = Field(None, description="MONAI Bundle metadata if available") + + @property + def display_name(self) -> str: + """Get a display-friendly name for the model. + + Returns the model's name if available, otherwise generates a + human-readable name from the model ID by removing the organization + prefix and converting underscores to spaces. + + Returns: + str: Display-friendly model name + """ + if self.name: + return self.name + return self.model_id.split("/")[-1].replace("_", " ").title() + + @property + def short_id(self) -> str: + """Get the short model ID without the organization prefix. + + Example: + 'MONAI/spleen_ct_segmentation' -> 'spleen_ct_segmentation' + + Returns: + str: Model ID without organization prefix + """ + return self.model_id.split("/")[-1] diff --git a/tools/pipeline-generator/pipeline_generator/generator/__init__.py b/tools/pipeline-generator/pipeline_generator/generator/__init__.py new file mode 100644 index 00000000..d16bf9b1 --- /dev/null +++ b/tools/pipeline-generator/pipeline_generator/generator/__init__.py @@ -0,0 +1,17 @@ +# Copyright 2025 MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Generator module for creating MONAI Deploy applications.""" + +from .app_generator import AppGenerator +from .bundle_downloader import BundleDownloader + +__all__ = ["AppGenerator", "BundleDownloader"] diff --git a/tools/pipeline-generator/pipeline_generator/generator/app_generator.py b/tools/pipeline-generator/pipeline_generator/generator/app_generator.py new file mode 100644 index 00000000..503bb1fd --- /dev/null +++ b/tools/pipeline-generator/pipeline_generator/generator/app_generator.py @@ -0,0 +1,601 @@ +# Copyright 2025 MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Generate MONAI Deploy applications from MONAI Bundles.""" + +import logging +import re +from pathlib import Path +from typing import Any, Dict, Optional + +from jinja2 import Environment, FileSystemLoader + +from ..config.settings import Settings, load_config +from .bundle_downloader import BundleDownloader + +logger = logging.getLogger(__name__) + + +class AppGenerator: + """Generates MONAI Deploy applications from MONAI Bundles.""" + + @staticmethod + def _sanitize_for_python_identifier(name: str) -> str: + """Sanitize a string to be a valid Python identifier. + + Args: + name: String to sanitize + + Returns: + Valid Python identifier + """ + # Replace invalid characters with underscores + sanitized = "".join(c if c.isalnum() or c == "_" else "_" for c in name) + + # Remove leading/trailing underscores + sanitized = sanitized.strip("_") + + # Ensure it doesn't start with a digit + if sanitized and sanitized[0].isdigit(): + sanitized = f"_{sanitized}" + + # Ensure it's not empty (all chars were invalid) + if not sanitized: + sanitized = "app" + + return sanitized + + def __init__(self, settings: Optional[Settings] = None) -> None: + """Initialize the generator. + + Args: + settings: Configuration settings (loads default if None) + """ + self.downloader = BundleDownloader() + self.settings = settings or load_config() + + # Set up Jinja2 template environment + template_dir = Path(__file__).parent.parent / "templates" + self.env = Environment( + loader=FileSystemLoader(str(template_dir)), + trim_blocks=True, + lstrip_blocks=True, + # Autoescape is intentionally disabled because we're generating + # Python code, YAML, and other non-HTML files. HTML escaping would + # break the generated code. Security is handled via input validation + # in generate_app() method. + autoescape=False, # nosec B701 + ) + + def generate_app( + self, + model_id: str, + output_dir: Path, + app_name: Optional[str] = None, + data_format: str = "auto", + ) -> Path: + """Generate a MONAI Deploy application from a HuggingFace model. + + Args: + model_id: HuggingFace model ID (e.g., 'MONAI/spleen_ct_segmentation') + output_dir: Directory to generate the application in + app_name: Optional custom application name + data_format: Data format - 'auto', 'dicom', or 'nifti' + + Returns: + Path to the generated application directory + """ + # Validate model_id to prevent code injection and path traversal + # Only allow model IDs like "owner/model-name" or "model_name", no leading/trailing slash, no "..", no empty segments + model_id_pattern = r"^(?!.*\.\.)(?!/)(?!.*//)(?!.*\/$)[A-Za-z0-9_-]+(\/[A-Za-z0-9_-]+)*$" + + if not model_id or not re.match(model_id_pattern, model_id): + raise ValueError( + ( + f"Invalid model_id: {model_id}. Only alphanumeric characters, hyphens, " + "underscores, and single slashes between segments are allowed. " + "No leading/trailing slashes, consecutive slashes, or '..' allowed." + ) + ) + + # Create output directory + output_dir.mkdir(parents=True, exist_ok=True) + + # Download the bundle + logger.info(f"Downloading bundle: {model_id}") + bundle_path = self.downloader.download_bundle(model_id, output_dir) + + # Organize bundle into proper structure if needed + self.downloader.organize_bundle_structure(bundle_path) + + # Read bundle metadata and config + metadata = self.downloader.get_bundle_metadata(bundle_path) + inference_config = self.downloader.get_inference_config(bundle_path) + + if not metadata: + logger.warning("No metadata.json found in bundle, using defaults") + metadata = self._get_default_metadata(model_id) + + if not inference_config: + logger.warning("No inference.json found in bundle, using defaults") + inference_config = {} + + # Detect model file + model_file = self.downloader.detect_model_file(bundle_path) + if model_file: + # Make path relative to bundle directory + model_file = model_file.relative_to(bundle_path) + + # Detect model type from model_id or metadata + model_type = self._detect_model_type(model_id, metadata) + + # Get model configuration if available + model_config = self.settings.get_model_config(model_id) + if model_config and data_format == "auto": + # Use data types from configuration + input_type = model_config.input_type + output_type = model_config.output_type + else: + # Fall back to detection + input_type = None + output_type = None + + # Prepare template context + context = self._prepare_context( + model_id=model_id, + metadata=metadata, + inference_config=inference_config, + model_file=model_file, + app_name=app_name, + data_format=data_format, + model_type=model_type, + input_type=input_type, + output_type=output_type, + model_config=model_config, + ) + + # Generate app.py + self._generate_app_py(output_dir, context) + + # Generate app.yaml + self._generate_app_yaml(output_dir, context) + + # Copy additional files if needed + self._copy_additional_files(output_dir, context) + + logger.info(f"Application generated successfully in: {output_dir}") + return output_dir + + def _prepare_context( + self, + model_id: str, + metadata: Dict[str, Any], + inference_config: Dict[str, Any], + model_file: Optional[Path], + app_name: Optional[str], + data_format: str = "auto", + model_type: str = "segmentation", + input_type: Optional[str] = None, + output_type: Optional[str] = None, + model_config: Optional[Any] = None, + ) -> Dict[str, Any]: + """Prepare context for template rendering. + + Args: + model_id: HuggingFace model ID + metadata: Bundle metadata + inference_config: Inference configuration + model_file: Path to model file relative to bundle + app_name: Optional custom application name + data_format: Data format - 'auto', 'dicom', or 'nifti' + + Returns: + Context dictionary for templates + """ + # Extract model name from ID + model_short_name = model_id.split("/")[-1] + + # Determine app name + if not app_name: + # For auto-generated names, apply title case after replacing underscores + # This ensures "test_model" becomes "TestModel" not "Test_Model" + title_name = model_short_name.replace("_", " ").replace("-", " ").title().replace(" ", "") + sanitized_name = self._sanitize_for_python_identifier(title_name) + app_name = f"{sanitized_name}App" + else: + # Ensure user-provided app_name is also a valid Python identifier + app_name = self._sanitize_for_python_identifier(app_name) + + # Determine task type from metadata + task = metadata.get("task", "segmentation").lower() + modality = metadata.get("modality", "CT").upper() + + # Extract network data format + network_data_format = metadata.get("network_data_format", {}) + inputs = network_data_format.get("inputs", {}) + outputs = network_data_format.get("outputs", {}) + + # Determine if this is DICOM or NIfTI based + if input_type: + # Use provided input type + use_dicom = input_type == "dicom" + use_image = input_type == "image" + elif data_format == "auto": + # Try to detect from inference config + use_dicom = self._detect_data_format(inference_config, modality) + use_image = False + elif data_format == "dicom": + use_dicom = True + use_image = False + else: # nifti + use_dicom = False + use_image = False + + # Extract organ/structure name + organ = self._extract_organ_name(model_short_name, metadata) + + # Get output postfix from inference config + output_postfix = "seg" # Default postfix + if "output_postfix" in inference_config: + postfix_value = inference_config["output_postfix"] + if isinstance(postfix_value, str) and not postfix_value.startswith("@"): + output_postfix = postfix_value + + # Resolve generator-level overrides/configs + resolved_channel_first = None + if model_config and getattr(model_config, "configs", None) is not None: + cfgs = model_config.configs + if isinstance(cfgs, list): + # Merge list of dicts; last one wins + merged = {} + for item in cfgs: + if isinstance(item, dict): + merged.update(item) + resolved_channel_first = merged.get("channel_first", None) + elif isinstance(cfgs, dict): + resolved_channel_first = cfgs.get("channel_first", None) + + # Determine final channel_first value + if resolved_channel_first is not None: + # Use explicit override from configuration + channel_first = resolved_channel_first + else: + # Apply default logic: False for image input classification, True otherwise + input_type_resolved = input_type or ("dicom" if use_dicom else ("image" if use_image else "nifti")) + if input_type_resolved == "image" and "classification" not in task.lower(): + channel_first = False + else: + channel_first = True + + # Collect dependency hints from metadata.json + required_packages_version = metadata.get("required_packages_version", {}) if metadata else {} + extra_dependencies = getattr(model_config, "dependencies", []) if model_config else [] + + # Handle dependency conflicts between config and metadata + config_deps = [] + if extra_dependencies: + # Extract dependency names from config overrides + config_deps = [dep.split(">=")[0].split("==")[0].split("<")[0] for dep in extra_dependencies] + + # Add metadata dependencies only if not overridden by config + if metadata and "numpy_version" in metadata and "numpy" not in config_deps: + extra_dependencies.append(f"numpy=={metadata['numpy_version']}") + if metadata and "pytorch_version" in metadata and "torch" not in config_deps: + extra_dependencies.append(f"torch=={metadata['pytorch_version']}") + + # Handle MONAI version - move logic from template to Python for better maintainability + has_monai_config = any(dep.startswith("monai") for dep in extra_dependencies) + if has_monai_config and metadata: + # Remove monai_version from metadata since we have config override + metadata = dict(metadata) # Make a copy + metadata.pop("monai_version", None) + elif not has_monai_config: + # No config MONAI dependency - add one based on metadata or fallback + if metadata and "monai_version" in metadata: + extra_dependencies.append(f"monai=={metadata['monai_version']}") + # Remove from metadata since it's now in extra_dependencies + metadata = dict(metadata) if metadata else {} + metadata.pop("monai_version", None) + else: + # No metadata version, use fallback + extra_dependencies.append("monai>=1.5.0") + + return { + "model_id": model_id, + "model_short_name": model_short_name, + "app_name": app_name, + "app_title": metadata.get("name", f"{organ} {task.title()} Inference"), + "app_description": metadata.get("description", ""), + "task": task, + "modality": modality, + "organ": organ, + "use_dicom": use_dicom, + "use_image": use_image, + "input_type": input_type or ("dicom" if use_dicom else "nifti"), + "output_type": output_type or ("json" if task == "classification" else "nifti"), + "model_file": str(model_file) if model_file else "models/model.ts", + "inference_config": inference_config, + "metadata": metadata, + "inputs": inputs, + "outputs": outputs, + "version": metadata.get("version", "1.0"), + "authors": metadata.get("authors", "MONAI"), + "output_postfix": output_postfix, + "model_type": model_type, + "channel_first": channel_first, + "required_packages_version": required_packages_version, + "extra_dependencies": extra_dependencies, + } + + def _detect_data_format(self, inference_config: Dict[str, Any], modality: str) -> bool: + """Detect whether to use DICOM or NIfTI based on inference config and modality. + + Args: + inference_config: Inference configuration + modality: Image modality + + Returns: + True for DICOM, False for NIfTI + """ + # Check preprocessing transforms for hints + if "preprocessing" in inference_config: + transforms = inference_config["preprocessing"].get("transforms", []) + # Handle case where transforms might be a string expression (e.g., "$@preprocessing_transforms + @deepedit_transforms") + if isinstance(transforms, str): + # If transforms is a string expression, we can't analyze it directly + # Look for LoadImaged in the inference config keys instead + config_str = str(inference_config) + if "LoadImaged" in config_str or "LoadImage" in config_str: + return False + elif isinstance(transforms, list): + for transform in transforms: + # Ensure transform is a dictionary before calling .get() + if isinstance(transform, dict): + target = transform.get("_target_", "") + if "LoadImaged" in target or "LoadImage" in target: + # This suggests NIfTI format + return False + + # Default based on modality + return modality in ["CT", "MR", "MRI"] + + def _extract_organ_name(self, model_name: str, metadata: Dict[str, Any]) -> str: + """Extract organ/structure name from model name or metadata. + + Args: + model_name: Short model name + metadata: Bundle metadata + + Returns: + Organ/structure name + """ + # Try to get from metadata first + if "organ" in metadata: + return str(metadata["organ"]) + + # Common organ names to extract + organs = [ + "spleen", + "liver", + "kidney", + "lung", + "brain", + "heart", + "pancreas", + "prostate", + "breast", + "colon", + ] + + model_lower = model_name.lower() + for organ in organs: + if organ in model_lower: + return organ.title() + + # Default + return "Organ" + + def _detect_model_type(self, model_id: str, metadata: Dict[str, Any]) -> str: + """Detect the model type based on model ID and metadata. + + Args: + model_id: HuggingFace model ID + metadata: Bundle metadata + + Returns: + Model type: segmentation, pathology, multimodal, multimodal_llm + """ + model_lower = model_id.lower() + + # Check for pathology models + if "exaonepath" in model_lower or "pathology" in model_lower: + return "pathology" + + # Check for multimodal LLMs + if "llama" in model_lower or "vila" in model_lower: + return "multimodal_llm" + + # Check for multimodal models + if "chat" in model_lower or "multimodal" in model_lower: + return "multimodal" + + # Check metadata for hints + if metadata: + task = metadata.get("task", "").lower() + if "pathology" in task: + return "pathology" + elif "chat" in task or "qa" in task: + return "multimodal" + + # Default to segmentation + return "segmentation" + + def _generate_app_py(self, output_dir: Path, context: Dict[str, Any]) -> None: + """Generate app.py file. + + Args: + output_dir: Output directory + context: Template context + """ + # Select template based on model type and input/output types + model_type = context.get("model_type", "segmentation") + input_type = context.get("input_type", "nifti") + output_type = context.get("output_type", "nifti") + + # Use the unified template for all cases + template = self.env.get_template("app.py.j2") + + app_content = template.render(**context) + app_path = output_dir / "app.py" + + with open(app_path, "w") as f: + f.write(app_content) + + # Make executable + app_path.chmod(0o755) + + logger.info(f"Generated app.py: {app_path}") + + def _generate_app_yaml(self, output_dir: Path, context: Dict[str, Any]) -> None: + """Generate app.yaml file. + + Args: + output_dir: Output directory + context: Template context + """ + template = self.env.get_template("app.yaml.j2") + yaml_content = template.render(**context) + + yaml_path = output_dir / "app.yaml" + with open(yaml_path, "w") as f: + f.write(yaml_content) + + logger.info(f"Generated app.yaml: {yaml_path}") + + def _copy_additional_files(self, output_dir: Path, context: Dict[str, Any]) -> None: + """Copy additional required files. + + Args: + output_dir: Output directory + context: Template context + """ + # Copy needed operators to generated application + self._copy_operators(output_dir, context) + + # Generate requirements.txt + self._generate_requirements(output_dir, context) + + # Generate README.md + self._generate_readme(output_dir, context) + + def _copy_operators(self, output_dir: Path, context: Dict[str, Any]) -> None: + """Copy needed operators to the generated application. + + Args: + output_dir: Output directory + context: Template context + """ + import shutil + + # Map operator usage based on context + needed_operators = [] + + input_type = context.get("input_type", "") + output_type = context.get("output_type", "") + task = context.get("task", "").lower() + + # Determine which operators are needed based on the application type + if input_type == "image": + needed_operators.extend(["generic_directory_scanner_operator.py", "image_file_loader_operator.py"]) + elif input_type == "custom": + needed_operators.extend( + ["llama3_vila_inference_operator.py", "prompts_loader_operator.py", "vlm_results_writer_operator.py"] + ) + elif input_type == "nifti": + needed_operators.append("generic_directory_scanner_operator.py") + + if output_type == "json": + needed_operators.append("json_results_writer_operator.py") + elif output_type == "image_overlay": + needed_operators.append("image_overlay_writer_operator.py") + elif output_type == "nifti": + needed_operators.append("nifti_writer_operator.py") + + if "classification" in task and input_type == "image": + needed_operators.append("monai_classification_operator.py") + + # Remove duplicates + needed_operators = list(set(needed_operators)) + + if needed_operators: + # Get the operators directory in templates + operators_dir = Path(__file__).parent.parent / "templates" / "operators" + + logger.info(f"Copying {len(needed_operators)} operators to generated application") + + for operator_file in needed_operators: + src_path = operators_dir / operator_file + if src_path.exists(): + dst_path = output_dir / operator_file + shutil.copy2(src_path, dst_path) + logger.debug(f"Copied operator: {operator_file}") + else: + logger.warning(f"Operator file not found: {src_path}") + + def _generate_requirements(self, output_dir: Path, context: Dict[str, Any]) -> None: + """Generate requirements.txt file. + + Args: + output_dir: Output directory + context: Template context + """ + template = self.env.get_template("requirements.txt.j2") + requirements_content = template.render(**context) + + requirements_path = output_dir / "requirements.txt" + with open(requirements_path, "w") as f: + f.write(requirements_content) + + logger.info(f"Generated requirements.txt: {requirements_path}") + + def _generate_readme(self, output_dir: Path, context: Dict[str, Any]) -> None: + """Generate README.md file. + + Args: + output_dir: Output directory + context: Template context + """ + template = self.env.get_template("README.md.j2") + readme_content = template.render(**context) + + readme_path = output_dir / "README.md" + with open(readme_path, "w") as f: + f.write(readme_content) + + logger.info(f"Generated README.md: {readme_path}") + + def _get_default_metadata(self, model_id: str) -> Dict[str, Any]: + """Get default metadata when none is provided. + + Args: + model_id: HuggingFace model ID + + Returns: + Default metadata dictionary + """ + model_name = model_id.split("/")[-1] + return { + "name": model_name.replace("_", " ").title(), + "version": "1.0", + "task": "segmentation", + "modality": "CT", + "description": f"MONAI Deploy application for {model_name}", + } diff --git a/tools/pipeline-generator/pipeline_generator/generator/bundle_downloader.py b/tools/pipeline-generator/pipeline_generator/generator/bundle_downloader.py new file mode 100644 index 00000000..ca468e15 --- /dev/null +++ b/tools/pipeline-generator/pipeline_generator/generator/bundle_downloader.py @@ -0,0 +1,269 @@ +# Copyright 2025 MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Download MONAI Bundles from HuggingFace.""" + +import json +import logging +from pathlib import Path +from typing import Any, Dict, Optional + +from huggingface_hub import HfApi, snapshot_download + +logger = logging.getLogger(__name__) + + +class BundleDownloader: + """Downloads MONAI Bundle files from HuggingFace.""" + + def __init__(self) -> None: + """Initialize the downloader.""" + self.api = HfApi() + + def download_bundle(self, model_id: str, output_dir: Path, cache_dir: Optional[Path] = None) -> Path: + """Download all files from a MONAI Bundle repository. + + Args: + model_id: HuggingFace model ID (e.g., 'MONAI/spleen_ct_segmentation') + output_dir: Directory to save the downloaded files + cache_dir: Optional cache directory for HuggingFace downloads + + Returns: + Path to the downloaded bundle directory + """ + logger.info(f"Downloading bundle: {model_id}") + + # Create output directory + bundle_dir = output_dir / "model" + bundle_dir.mkdir(parents=True, exist_ok=True) + + try: + # Download all files from the repository + local_path = snapshot_download( + repo_id=model_id, + local_dir=bundle_dir, + cache_dir=cache_dir, + ) + + logger.info(f"Bundle downloaded to: {local_path}") + return Path(local_path) + + except Exception as e: + logger.error(f"Failed to download bundle {model_id}: {e}") + raise + + def get_bundle_metadata(self, bundle_path: Path) -> Optional[Dict[str, Any]]: + """Read metadata.json from downloaded bundle. + + Args: + bundle_path: Path to the downloaded bundle + + Returns: + Dictionary containing bundle metadata or None if not found + """ + metadata_paths = [ + bundle_path / "metadata.json", + bundle_path / "configs" / "metadata.json", + ] + + for metadata_path in metadata_paths: + if metadata_path.exists(): + try: + with open(metadata_path, "r") as f: + data: Dict[str, Any] = json.load(f) + return data + except Exception as e: + logger.error(f"Failed to read metadata from {metadata_path}: {e}") + + return None + + def get_inference_config(self, bundle_path: Path) -> Optional[Dict[str, Any]]: + """Read inference.json from downloaded bundle. + + Args: + bundle_path: Path to the downloaded bundle + + Returns: + Dictionary containing inference configuration or None if not found + """ + inference_paths = [ + bundle_path / "inference.json", + bundle_path / "configs" / "inference.json", + ] + + for inference_path in inference_paths: + if inference_path.exists(): + try: + with open(inference_path, "r") as f: + data: Dict[str, Any] = json.load(f) + return data + except Exception as e: + logger.error(f"Failed to read inference config from {inference_path}: {e}") + + return None + + def detect_model_file(self, bundle_path: Path) -> Optional[Path]: + """Detect the model file in the bundle. + + Args: + bundle_path: Path to the downloaded bundle + + Returns: + Path to the model file or None if not found + """ + # Common model file patterns + model_patterns = [ + "models/model.ts", # TorchScript + "models/model.pt", # PyTorch + "models/model.onnx", # ONNX + "model.ts", + "model.pt", + "model.onnx", + ] + + for pattern in model_patterns: + model_path = bundle_path / pattern + if model_path.exists(): + logger.info(f"Found model file: {model_path}") + return model_path + + # If no standard pattern found, search for any model file + for ext in [".ts", ".pt", ".onnx"]: + model_files = list(bundle_path.glob(f"**/*{ext}")) + if model_files: + logger.info(f"Found model file: {model_files[0]}") + return model_files[0] + + logger.warning(f"No model file found in bundle: {bundle_path}") + return None + + def organize_bundle_structure(self, bundle_path: Path) -> None: + """Organize bundle files into the expected MONAI Bundle structure. + + Creates the standard structure if files are in the root directory: + bundle_root/ + configs/ + metadata.json + inference.json + models/ + model.pt + model.ts + + Args: + bundle_path: Path to the downloaded bundle + """ + configs_dir = bundle_path / "configs" + models_dir = bundle_path / "models" + + # Check if structure already exists + has_configs_structure = configs_dir.exists() and (configs_dir / "metadata.json").exists() + has_models_structure = models_dir.exists() and any(models_dir.glob("model.*")) + + if has_configs_structure and has_models_structure: + logger.debug("Bundle already has proper structure") + return + + logger.info("Organizing bundle into standard structure") + + # Create directories + configs_dir.mkdir(exist_ok=True) + models_dir.mkdir(exist_ok=True) + + # Move config files to configs/ + config_files = ["metadata.json", "inference.json"] + for config_file in config_files: + src_path = bundle_path / config_file + if src_path.exists() and not (configs_dir / config_file).exists(): + src_path.rename(configs_dir / config_file) + logger.debug(f"Moved {config_file} to configs/") + + # Move model files to models/ + # Prefer PyTorch (.pt) > ONNX (.onnx) > TorchScript (.ts) for better compatibility + model_extensions = [".pt", ".onnx", ".ts"] + + # First move model files from root directory + for ext in model_extensions: + for model_file in bundle_path.glob(f"*{ext}"): + if model_file.is_file() and not (models_dir / model_file.name).exists(): + model_file.rename(models_dir / model_file.name) + logger.debug(f"Moved {model_file.name} to models/") + + # Check if we already have a suitable model in the main directory + # Prefer .pt files, then .onnx, then .ts + has_suitable_model = False + for ext in model_extensions: + if any(models_dir.glob(f"*{ext}")): + has_suitable_model = True + break + + # If no suitable model in main directory, move from subdirectories + if not has_suitable_model: + # Also move model files from subdirectories to the main models/ directory + # This handles cases where models are in subdirectories like models/A100/ + # Prefer PyTorch models over TensorRT models for better compatibility + for ext in model_extensions: + model_files = list(models_dir.glob(f"**/*{ext}")) + if not model_files: + continue + + # Filter files that are not in the main models directory + subdirectory_files = [f for f in model_files if f.parent != models_dir] + if not subdirectory_files: + continue + + target_name = f"model{ext}" + target_path = models_dir / target_name + if target_path.exists(): + continue # Target already exists + + # Prefer non-TensorRT models for better compatibility + # TensorRT models often have "_trt" in their name + preferred_file = None + for model_file in subdirectory_files: + if "_trt" not in model_file.name.lower(): + preferred_file = model_file + break + + # If no non-TensorRT model found, use the first available + if preferred_file is None: + preferred_file = subdirectory_files[0] + + # Move the preferred model file + preferred_file.rename(target_path) + logger.debug(f"Moved {preferred_file.name} from {preferred_file.parent.name}/ to models/{target_name}") + + # Clean up empty subdirectory if it exists + try: + if preferred_file.parent.exists() and not any(preferred_file.parent.iterdir()): + preferred_file.parent.rmdir() + logger.debug(f"Removed empty directory {preferred_file.parent}") + except OSError: + pass # Directory not empty or other issue + break # Only move one model file total + + # Ensure we have model.pt or model.ts in the main directory for MONAI Deploy + # Create symlinks with standard names if needed + standard_model_path = models_dir / "model.pt" + if not standard_model_path.exists(): + # Look for any .pt file to link to model.pt + pt_files = list(models_dir.glob("*.pt")) + if pt_files: + # Create a copy with the standard name + pt_files[0].rename(standard_model_path) + logger.debug(f"Renamed {pt_files[0].name} to model.pt") + else: + # No .pt file found, look for .ts file and create model.ts instead + standard_ts_path = models_dir / "model.ts" + if not standard_ts_path.exists(): + ts_files = list(models_dir.glob("*.ts")) + if ts_files: + ts_files[0].rename(standard_ts_path) + logger.debug(f"Renamed {ts_files[0].name} to model.ts") diff --git a/tools/pipeline-generator/pipeline_generator/templates/README.md.j2 b/tools/pipeline-generator/pipeline_generator/templates/README.md.j2 new file mode 100644 index 00000000..58b3c3bb --- /dev/null +++ b/tools/pipeline-generator/pipeline_generator/templates/README.md.j2 @@ -0,0 +1,214 @@ +# {{ app_title }} + +Generated from HuggingFace model: [{{ model_id }}](https://huggingface.co/{{ model_id }}) + +## Model Information + +**Task**: {{ task|title }} +**Modality**: {{ modality }} +**Network**: {{ metadata.get('network_data_format', {}).get('network', 'Unknown') }} +{% if model_type %}**Model Type**: {{ model_type|replace('_', ' ')|title }}{% endif %} + +{{ app_description }} + +{% if model_type == "pathology" %} +### Pathology-Specific Features + +This application includes: +- **Stain Normalization**: Applies Macenko stain normalization to input images for consistent processing +- **Optimized for Pathology**: Designed to handle whole slide images and pathology-specific preprocessing + +{% elif model_type == "multimodal" %} +### Multimodal Features + +This application supports: +- **Image Analysis**: Processes medical images for feature extraction +- **Text Integration**: Can accept text prompts for guided analysis +- **Report Generation**: Produces structured reports from the analysis + +{% elif model_type == "multimodal_llm" %} +### Multimodal LLM Features + +This application provides: +- **Vision-Language Integration**: Combines image understanding with language generation +- **Natural Language Output**: Generates human-readable descriptions and analysis +- **Interactive Queries**: Supports text prompts for specific questions about the images +- **Clinical Report Generation**: Can produce detailed medical reports + +{% endif %} + +## Requirements + +### Option 1: Using uv (Recommended) + +If you're running from the pipeline generator directory: + +```bash +# Commands should be run with uv +uv run pg run . --input /path/to/input --output /path/to/output +``` + +### Option 2: Using Virtual Environment + +Create and activate a virtual environment (optional but recommended): + +```bash +# Create virtual environment +python -m venv venv + +# Activate virtual environment +# On Linux/Mac: +source venv/bin/activate +# On Windows: +# venv\Scripts\activate + +# Install dependencies +pip install -r requirements.txt +``` + +**Note**: For directory-based bundle support, you may need to use a local modified version of MONAI Deploy App SDK: +```bash +pip install -e /path/to/monai-deploy-app-sdk +``` + +## Usage + +### Running the Application + +#### Option 1: Using Pipeline Generator with uv + +From the pipeline generator directory: + +```bash +uv run pg run . --input /path/to/input --output /path/to/output +``` + +This command will automatically: +- Create a virtual environment +- Install all dependencies +- Run the application + +#### Option 2: Using Pipeline Generator Directly + +If you have the Pipeline Generator installed globally: + +```bash +pg run . --input /path/to/input --output /path/to/output +``` + +#### Option 3: Manual Execution + +```bash +# If using virtual environment (recommended) +# Activate it first: +# source venv/bin/activate # Linux/Mac +# venv\Scripts\activate # Windows + +# Run the application +python app.py -i /path/to/input -o /path/to/output +``` + +**Input**: +{% if use_dicom %} +- Directory containing DICOM series +{% else %} +- Directory containing NIfTI files (.nii or .nii.gz) +{% endif %} + +**Output**: +{% if use_dicom %} +- DICOM Segmentation objects +- (Optional) STL mesh files +{% else %} +- NIfTI segmentation files +{% endif %} + +### Command Line Arguments + +- `-i, --input`: Input data directory (required) +- `-o, --output`: Output directory (default: ./output) +- `-m, --model`: Path to model/bundle directory (default: ./model) + +### Examples + +```bash +# Using pg with uv (from pipeline generator directory) +uv run pg run . --input ./test_data --output ./results + +# Using pg directly (if installed globally) +pg run . --input ./test_data --output ./results + +# Manual execution +python app.py -i ./test_data -o ./results + +# Use a different model location +python app.py -i ./test_data -o ./results -m /path/to/model +``` + +## Application Structure + +``` +. +├── app.py # Main application file +├── app.yaml # Application configuration +├── requirements.txt # Python dependencies +└── model/ # MONAI Bundle + ├── configs/ # Bundle configurations + │ ├── metadata.json + │ └── inference.json + └── models/ # Model weights + └── model.{{ 'ts' if model_file and model_file.endswith('.ts') else 'pt' }} +``` + +## Deployment + +### Local Deployment + +Run directly using Python as shown above. + +### Container Deployment + +Package the application as a container using Holoscan CLI: + +```bash +# Package for x64 workstations +holoscan package app -c app.yaml --platform linux/amd64 -t {{ model_short_name|lower }}:latest + +# Package for IGX Orin devkits +holoscan package app -c app.yaml --platform linux/arm64 -t {{ model_short_name|lower }}:latest +``` + +Run the containerized application: + +```bash +# Using Holoscan CLI +holoscan run -i /path/to/input -o /path/to/output {{ model_short_name|lower }}:latest + +# Or using Docker directly +docker run -v /path/to/input:/input -v /path/to/output:/output {{ model_short_name|lower }}:latest +``` + +## Model Details + +{% if metadata %} +### Metadata +- **Version**: {{ metadata.get('version', 'Unknown') }} +- **Authors**: {{ metadata.get('authors', 'Unknown') }} +- **License**: {{ metadata.get('license', 'See model page') }} +{% if metadata.get('intended_use') %} +- **Intended Use**: {{ metadata.get('intended_use') }} +{% endif %} +{% endif %} + +### Network Architecture +{% if metadata.get('network_data_format') %} +- **Input Shape**: {{ metadata.get('network_data_format', {}).get('inputs', {}) }} +- **Output Shape**: {{ metadata.get('network_data_format', {}).get('outputs', {}) }} +{% endif %} + +For more details, visit the model page: [{{ model_id }}](https://huggingface.co/{{ model_id }}) + +## License + +This application is generated using the MONAI Deploy Pipeline Generator. +Please refer to the model's license for usage restrictions. diff --git a/tools/pipeline-generator/pipeline_generator/templates/app.py.j2 b/tools/pipeline-generator/pipeline_generator/templates/app.py.j2 new file mode 100644 index 00000000..c43b6334 --- /dev/null +++ b/tools/pipeline-generator/pipeline_generator/templates/app.py.j2 @@ -0,0 +1,378 @@ +#!/usr/bin/env python3 +# Copyright 2025 MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +{{ app_title }} + +Generated from HuggingFace model: {{ model_id }} +{{ app_description }} +""" + +import logging +import os +from pathlib import Path + +{% if use_dicom %} +# Required for setting SegmentDescription attributes. Direct import as this is not part of App SDK package. +from pydicom.sr.codedict import codes + +from monai.deploy.conditions import CountCondition + +{% endif %} +from monai.deploy.core import AppContext, Application +from monai.deploy.core.domain import Image +from monai.deploy.core.io_type import IOType + +{% if use_dicom %} +from monai.deploy.operators.dicom_data_loader_operator import DICOMDataLoaderOperator +from monai.deploy.operators.dicom_seg_writer_operator import DICOMSegmentationWriterOperator, SegmentDescription +from monai.deploy.operators.dicom_series_selector_operator import DICOMSeriesSelectorOperator +from monai.deploy.operators.dicom_series_to_volume_operator import DICOMSeriesToVolumeOperator + +{% if 'segmentation' in task.lower() %} +from monai.deploy.operators.stl_conversion_operator import STLConversionOperator + +{% endif %} +{% elif input_type == "image" %} +from generic_directory_scanner_operator import GenericDirectoryScanner +from image_file_loader_operator import ImageFileLoader + +{% elif input_type == "custom" %} +from llama3_vila_inference_operator import Llama3VILAInferenceOperator + +# Custom operators for vision-language models +from prompts_loader_operator import PromptsLoaderOperator +from vlm_results_writer_operator import VLMResultsWriterOperator + +{% else %} +from generic_directory_scanner_operator import GenericDirectoryScanner + +from monai.deploy.operators.nii_data_loader_operator import NiftiDataLoader + +{% endif %} +{% if output_type == "json" %} +from json_results_writer_operator import JSONResultsWriter + +{% elif output_type == "image_overlay" %} +from image_overlay_writer_operator import ImageOverlayWriter + +{% elif not use_dicom and input_type != "custom" %} +from nifti_writer_operator import NiftiWriter + +{% endif %} +{% if "classification" in task.lower() and input_type == "image" %} +from monai_classification_operator import MonaiClassificationOperator + +{% elif not (input_type == "custom" and output_type == "custom") %} +from monai.deploy.operators.monai_bundle_inference_operator import ( + BundleConfigNames, + IOMapping, + MonaiBundleInferenceOperator, +) + +{% endif %} + + +class {{ app_name }}(Application): + """MONAI Deploy application for {{ app_title }} using a MONAI Bundle. + + {% if use_dicom %} + This application loads a set of DICOM instances, selects the appropriate series, converts the series to + 3D volume image, performs inference with the built-in MONAI Bundle inference operator, including pre-processing + and post-processing, saves the segmentation image in a DICOM Seg OID in an instance file{% if 'segmentation' in task.lower() %}, and optionally the + surface mesh in STL format{% endif %}. + + Pertinent MONAI Bundle: {{ model_id }} + {% elif input_type == "image" and output_type == "json" %} + This application processes common image formats (JPEG, PNG, etc.) and outputs + classification results as JSON files. + {% elif input_type == "custom" and output_type == "custom" %} + This application processes prompts and images using a vision-language model (VLM). + Prompts are specified in a prompts.yaml file and can include tasks such as visual question answering, image captioning, or visual reasoning. + Each prompt entry may contain a textual question or instruction and an associated image path. + The application generates outputs such as text answers, captions, or image overlays, depending on the prompt and the configured output type. + {% else %} + This application follows the pipeline structure: + [GenericDirectoryScanner] → [{{ 'ImageFileLoader' if input_type == 'image' else 'NiftiDataLoader' }}] → [Preprocessing Op] → [Inference Op] → [Postprocessing Op] → [Sink/{{ 'JSONResultsWriter' if output_type == 'json' else 'NiftiWriter' }}] + + The GenericDirectoryScanner finds files with appropriate extensions, + the file loader processes individual files, and the MonaiBundleInferenceOperator + handles preprocessing, inference, and postprocessing based on configurations + loaded dynamically from inference.json. + {% endif %} + """ + + def __init__(self, *args, **kwargs): + """Creates an application instance.""" + self._logger = logging.getLogger("{}.{}".format(__name__, type(self).__name__)) + super().__init__(*args, **kwargs) + + def run(self, *args, **kwargs): + # This method calls the base class to run + self._logger.info(f"Begin {self.run.__name__}") + super().run(*args, **kwargs) + self._logger.info(f"End {self.run.__name__}") + + def compose(self): + """Creates the app specific operators and chain them up in the processing DAG.""" + + self._logger.info(f"Begin {self.compose.__name__}") + + # Use Commandline options over environment variables to init context + app_context: AppContext = Application.init_app_context(self.argv) + app_input_path = Path(app_context.input_path) + app_output_path = Path(app_context.output_path) + + # Set the bundle path from environment variable or use default + bundle_path = os.environ.get("BUNDLE_PATH", str(Path(__file__).parent / "model")) + bundle_path = Path(bundle_path) + if not bundle_path.exists(): + self._logger.warning(f"Bundle path does not exist: {bundle_path}") + + # Create operators + {% if use_dicom %} + # Create the custom operator(s) as well as SDK built-in operator(s) + study_loader_op = DICOMDataLoaderOperator( + self, CountCondition(self, 1), input_folder=app_input_path, name="study_loader_op" + ) + series_selector_op = DICOMSeriesSelectorOperator(self, rules=Sample_Rules_Text, name="series_selector_op") + series_to_vol_op = DICOMSeriesToVolumeOperator(self, name="series_to_vol_op") + {% elif input_type == "image" %} + # Image processing using chained operators + # Scanner finds all image files in the directory + scanner_op = GenericDirectoryScanner( + self, + input_folder=app_input_path, + file_extensions=['.jpg', '.jpeg', '.png', '.bmp', '.tiff', '.tif'], + recursive=True, + name="image_scanner" + ) + + # Loader processes individual image files + # For 2D RGB bundles that include EnsureChannelFirstd(channel_dim=-1) in preprocessing, + # emit HWC arrays to let the bundle handle channel movement. + loader_op = ImageFileLoader( + self, + channel_first={{ channel_first }}, + name="image_loader" + ) + {% elif input_type == "custom" %} + # Prompts loader for vision-language models + loader_op = PromptsLoaderOperator( + self, + input_folder=app_input_path, + name="prompts_loader" + ) + {% else %} + # NIfTI processing using chained operators + # Scanner finds all NIfTI files in the directory + scanner_op = GenericDirectoryScanner( + self, + input_folder=app_input_path, + file_extensions=['.nii', '.nii.gz'], + recursive=True, + name="nifti_scanner" + ) + + # Loader processes individual NIfTI files + loader_op = NiftiDataLoader( + self, + input_path=None, # Will be provided by scanner + name="nifti_loader" + ) + {% endif %} + + {% if input_type == "custom" and output_type == "custom" %} + # Vision-language model inference operator + inference_op = Llama3VILAInferenceOperator( + self, + app_context=app_context, + model_path=bundle_path, + name="vlm_inference" + ) + {% elif "classification" in task.lower() and input_type == "image" %} + # MonaiClassificationOperator for classification models + # The bundle path can be overridden with -m argument at runtime + inference_op = MonaiClassificationOperator( + self, + app_context=app_context, + bundle_path=Path(bundle_path), + name="classification" + ) + {% else %} + # MonaiBundleInferenceOperator with dynamic config loading + # The bundle path can be overridden with -m argument at runtime + {% if use_dicom %} + config_names = BundleConfigNames(config_names=["inference"]) # Same as the default + {% endif %} + inference_op = MonaiBundleInferenceOperator( + self, + input_mapping=[IOMapping("image", Image, IOType.IN_MEMORY)], + output_mapping=[IOMapping("pred", Image, IOType.IN_MEMORY)], + app_context=app_context, + bundle_path=Path(bundle_path), + {% if use_dicom %}bundle_config_names=config_names,{% endif %} + name="bundle_inference{% if use_dicom %}_op{% endif %}" + ) + {% endif %} + + {% if use_dicom and 'segmentation' in task.lower() %} + # Create DICOM Seg writer providing the required segment description for each segment + segment_descriptions = [ + SegmentDescription( + segment_label="{{ organ }}", + segmented_property_category=codes.SCT.Organ, + segmented_property_type=codes.SCT.{{ organ }}, + algorithm_name="volumetric (3D) segmentation of the {{ organ|lower }} from {{ modality }} image", + algorithm_family=codes.DCM.ArtificialIntelligence, + algorithm_version="{{ version }}", + ) + ] + + custom_tags = {"SeriesDescription": "AI generated Seg, not for clinical use."} + + dicom_seg_writer = DICOMSegmentationWriterOperator( + self, + segment_descriptions=segment_descriptions, + custom_tags=custom_tags, + output_folder=app_output_path, + name="dicom_seg_writer", + ) +{% elif output_type == "json" %} + # JSON results writer that saves classification results + writer_op = JSONResultsWriter( + self, + output_folder=app_output_path, + name="json_writer" + ) +{% elif output_type == "image_overlay" %} + # Overlay writer to blend segmentation predictions on original images + writer_op = ImageOverlayWriter( + self, + output_folder=app_output_path, + name="overlay_writer" + ) +{% elif output_type == "custom" %} + # VLM results writer for custom outputs + writer_op = VLMResultsWriterOperator( + self, + output_folder=app_output_path, + name="vlm_writer" + ) +{% elif not use_dicom %} + # NIfTI writer that saves results with proper naming from bundle config + writer_op = NiftiWriter( + self, + output_folder=app_output_path, + output_postfix="{{ output_postfix }}", # Postfix from bundle config + name="nifti_writer" + ) + {% endif %} + + # Connect operators in the pipeline + {% if use_dicom %} + # Create the processing pipeline, by specifying the source and destination operators, and + # ensuring the output from the former matches the input of the latter, in both name and type + self.add_flow(study_loader_op, series_selector_op, {("dicom_study_list", "dicom_study_list")}) + self.add_flow( + series_selector_op, series_to_vol_op, {("study_selected_series_list", "study_selected_series_list")} + ) + self.add_flow(series_to_vol_op, inference_op, {("image", "image")}) + + {% if 'segmentation' in task.lower() %} + # Note below the dicom_seg_writer requires two inputs, each coming from a source operator + self.add_flow( + series_selector_op, dicom_seg_writer, {("study_selected_series_list", "study_selected_series_list")} + ) + self.add_flow(inference_op, dicom_seg_writer, {("pred", "seg_image")}) + + # Create the surface mesh STL conversion operator and add it to the app execution flow + stl_conversion_op = STLConversionOperator( + self, output_file=app_output_path.joinpath("stl/{{ organ|lower }}.stl"), name="stl_conversion_op" + ) + self.add_flow(inference_op, stl_conversion_op, {("pred", "image")}) + {% endif %} +{% elif input_type == "custom" and output_type == "custom" %} + # Connect prompts loader to inference operator + self.add_flow(loader_op, inference_op, { + ("image", "image"), + ("prompt", "prompt"), + ("output_type", "output_type"), + ("request_id", "request_id"), + ("generation_params", "generation_params") + }) + # Connect inference operator to results writer + self.add_flow(inference_op, writer_op, { + ("result", "result"), + ("output_type", "output_type"), + ("request_id", "request_id") + }) +{% else %} + # Connect scanner to loader for both image and nifti cases + {% if input_type == "image" %} + self.add_flow(scanner_op, loader_op, {("file_path", "file_path")}) + {% else %} + self.add_flow(scanner_op, loader_op, {("file_path", "image_path")}) + {% endif %} + self.add_flow(loader_op, inference_op, {("image", "image")}) + {% if output_type == 'json' %} + self.add_flow(inference_op, writer_op, {("pred", "pred")}) + self.add_flow(scanner_op, writer_op, {("filename", "filename")}) + {% elif output_type == 'image_overlay' %} + # Connect both original image and prediction to overlay writer + self.add_flow(loader_op, writer_op, {("image", "image")}) + self.add_flow(scanner_op, writer_op, {("filename", "filename")}) + self.add_flow(inference_op, writer_op, {("pred", "pred")}) + {% else %} + self.add_flow(inference_op, writer_op, {("pred", "image")}) + self.add_flow(scanner_op, writer_op, {("filename", "filename")}) + {% endif %} +{% endif %} + + self._logger.info(f"End {self.compose.__name__}") + + +{% if use_dicom %} +# This is a sample series selection rule in JSON, simply selecting {{ modality }} series +# If the study has more than 1 {{ modality }} series, then all of them will be selected +Sample_Rules_Text = """ +{ + "selections": [ + { + "name": "{{ modality }} Series", + "conditions": { + "StudyDescription": "(.*?)", + "Modality": "(?i){{ modality }}", + "SeriesDescription": "(.*?)" + } + } + ] +} +""" +{% endif %} + +if __name__ == "__main__": + # Creates the app and test it standalone. When running is this mode, please note the following: + # -m , for model file path + {% if use_dicom %} + # -i , for input DICOM {{ modality }} series folder + {% else %} + # -i , for input folder path + {% endif %} + # -o , for output folder path, default $PWD/output + # e.g. + # python app.py -i /path/to/input -o /path/to/output -m /path/to/bundle + # + logging.basicConfig(level=logging.INFO) + logging.info(f"Begin {__name__}") + {{ app_name }}().run() + logging.info(f"End {__name__}") diff --git a/tools/pipeline-generator/pipeline_generator/templates/app.yaml.j2 b/tools/pipeline-generator/pipeline_generator/templates/app.yaml.j2 new file mode 100644 index 00000000..8fdcae02 --- /dev/null +++ b/tools/pipeline-generator/pipeline_generator/templates/app.yaml.j2 @@ -0,0 +1,15 @@ +--- +# Generated MONAI Deploy App Package Configuration +# Model: {{ model_id }} + +application: + title: {{ app_title }} + version: {{ version }} + inputFormats: ["file"] + outputFormats: ["file"] + +resources: + cpu: 1 + gpu: 1 + memory: 1Gi + gpuMemory: 7Gi diff --git a/tools/pipeline-generator/pipeline_generator/templates/operators/generic_directory_scanner_operator.py b/tools/pipeline-generator/pipeline_generator/templates/operators/generic_directory_scanner_operator.py new file mode 100644 index 00000000..de7350f7 --- /dev/null +++ b/tools/pipeline-generator/pipeline_generator/templates/operators/generic_directory_scanner_operator.py @@ -0,0 +1,200 @@ +# Copyright 2025 MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from pathlib import Path +from typing import List, Union + +from monai.deploy.core import ConditionType, Fragment, Operator, OperatorSpec + + +class GenericDirectoryScanner(Operator): + """Scan a directory for files matching specified extensions and emit file paths one by one. + + This operator provides a generic way to iterate through files in a directory, + emitting one file path at a time. It can be chained with file-specific loaders + to create flexible data loading pipelines. + + Named Outputs: + file_path: Path to the current file being processed + filename: Name of the current file (without extension) + file_index: Current file index (0-based) + total_files: Total number of files found + """ + + def __init__( + self, + fragment: Fragment, + *args, + input_folder: Union[str, Path], + file_extensions: List[str], + recursive: bool = True, + case_sensitive: bool = False, + **kwargs, + ) -> None: + """Initialize the GenericDirectoryScanner. + + Args: + fragment: An instance of the Application class + input_folder: Path to folder containing files to scan + file_extensions: List of file extensions to scan for (e.g., ['.jpg', '.png']) + recursive: If True, scan subdirectories recursively + case_sensitive: If True, perform case-sensitive extension matching + """ + self._logger = logging.getLogger("{}.{}".format(__name__, type(self).__name__)) + self._input_folder = Path(input_folder) + self._file_extensions = [ext if ext.startswith(".") else f".{ext}" for ext in file_extensions] + self._recursive = bool(recursive) + self._case_sensitive = bool(case_sensitive) + + # State tracking + self._files: list[Path] = [] + self._current_index = 0 + + super().__init__(fragment, *args, **kwargs) + + def _find_files(self) -> List[Path]: + """Find all files matching the specified extensions.""" + files = [] + + # Normalize extensions for comparison + if not self._case_sensitive: + extensions = [ext.lower() for ext in self._file_extensions] + else: + extensions = self._file_extensions + + # Choose search method based on recursive flag + if self._recursive: + search_pattern = "**/*" + search_method = self._input_folder.rglob + else: + search_pattern = "*" + search_method = self._input_folder.glob + + # Find all files and filter by extension + for file_path in search_method(search_pattern): + if file_path.is_file(): + # Skip hidden files (starting with .) to avoid macOS metadata files like ._file.nii.gz + if file_path.name.startswith("."): + continue + + # Handle compound extensions like .nii.gz by checking if filename ends with any extension + filename = file_path.name + if not self._case_sensitive: + filename = filename.lower() + + # Check if filename ends with any of the specified extensions + for ext in extensions: + if filename.endswith(ext): + files.append(file_path) + break # Only add once even if multiple extensions match + + # Sort files for consistent ordering + files.sort() + return files + + def setup(self, spec: OperatorSpec): + """Define the operator outputs.""" + spec.output("file_path") + spec.output("filename") + spec.output("file_index").condition(ConditionType.NONE) + spec.output("total_files").condition(ConditionType.NONE) + + # Pre-initialize the files list + if not self._input_folder.is_dir(): + raise ValueError(f"Input folder {self._input_folder} is not a directory") + + self._files = self._find_files() + self._current_index = 0 + + if not self._files: + self._logger.warning(f"No files found in {self._input_folder} with extensions {self._file_extensions}") + else: + self._logger.info(f"Found {len(self._files)} files to process with extensions {self._file_extensions}") + + def compute(self, op_input, op_output, context): + """Emit the next file path.""" + + # Check if we have more files to process + if self._current_index >= len(self._files): + # No more files to process + self._logger.info("All files have been processed") + self.fragment.stop_execution() + return + + # Get the current file path + file_path = self._files[self._current_index] + + try: + # Emit file information + op_output.emit(str(file_path), "file_path") + op_output.emit(file_path.stem, "filename") + op_output.emit(self._current_index, "file_index") + op_output.emit(len(self._files), "total_files") + + self._logger.info(f"Emitted file: {file_path.name} ({self._current_index + 1}/{len(self._files)})") + + except Exception as e: + self._logger.error(f"Failed to process file {file_path}: {e}") + + # Move to the next file + self._current_index += 1 + + +def test(): + """Test the GenericDirectoryScanner operator.""" + import tempfile + + # Create a temporary directory with test files + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + + # Create test files with different extensions + test_files = ["test1.jpg", "test2.png", "test3.nii", "test4.nii.gz", "test5.txt", "test6.jpeg"] + + for filename in test_files: + (temp_path / filename).touch() + + # Create a subdirectory with more files + sub_dir = temp_path / "subdir" + sub_dir.mkdir() + (sub_dir / "sub_test.jpg").touch() + (sub_dir / "sub_test.nii").touch() + + # Test the operator with image extensions + fragment = Fragment() + scanner = GenericDirectoryScanner( + fragment, input_folder=temp_path, file_extensions=[".jpg", ".jpeg", ".png"], recursive=True + ) + + # Simulate setup + from monai.deploy.core import OperatorSpec + + spec = OperatorSpec() + scanner.setup(spec) + + print(f"Found {len(scanner._files)} image files") + + # Simulate compute calls + class MockOutput: + def emit(self, data, name): + print(f"Emitted {name}: {data}") + + mock_output = MockOutput() + + # Process a few files + for i in range(min(3, len(scanner._files))): + print(f"\n--- Processing file {i + 1} ---") + scanner.compute(None, mock_output, None) + + +if __name__ == "__main__": + test() diff --git a/tools/pipeline-generator/pipeline_generator/templates/operators/image_file_loader_operator.py b/tools/pipeline-generator/pipeline_generator/templates/operators/image_file_loader_operator.py new file mode 100644 index 00000000..266d5c64 --- /dev/null +++ b/tools/pipeline-generator/pipeline_generator/templates/operators/image_file_loader_operator.py @@ -0,0 +1,193 @@ +# Copyright 2025 MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from pathlib import Path +from typing import Optional + +import numpy as np + +from monai.deploy.core import ConditionType, Fragment, Image, Operator, OperatorSpec +from monai.deploy.utils.importutil import optional_import + +PILImage, _ = optional_import("PIL", name="Image") + + +# @md.env(pip_packages=["Pillow >= 8.0.0"]) +class ImageFileLoader(Operator): + """Load a single image file (JPEG, PNG, BMP, TIFF) and convert to Image object. + + This operator loads a single image file specified via input path and outputs an Image object. + It can be chained with GenericDirectoryScanner for batch processing of multiple images. + + By default it outputs channel-first arrays (CHW) to match many MONAI pipelines. For 2D RGB models + whose bundle preprocessing includes EnsureChannelFirstd(channel_dim=-1), set ``channel_first=False`` + to emit HWC arrays so the bundle transform handles channel movement. + + Named Inputs: + file_path: Path to the image file to load (optional, overrides input_path) + + Named Outputs: + image: Image object loaded from file + filename: Name of the loaded file (without extension) + """ + + SUPPORTED_EXTENSIONS = [".jpg", ".jpeg", ".png", ".bmp", ".tiff", ".tif"] + + def __init__( + self, + fragment: Fragment, + *args, + input_path: Optional[Path] = None, + channel_first: bool = True, + **kwargs, + ) -> None: + """Initialize the ImageFileLoader. + + Args: + fragment: An instance of the Application class + input_path: Default path to image file (can be overridden by input) + channel_first: If True (default), emit CHW arrays. If False, emit HWC arrays. + """ + self._logger = logging.getLogger("{}.{}".format(__name__, type(self).__name__)) + self._input_path = Path(input_path) if input_path else None + self._channel_first = bool(channel_first) + + # Port names + self._input_name_path = "file_path" + self._output_name_image = "image" + self._output_name_filename = "filename" + + super().__init__(fragment, *args, **kwargs) + + def setup(self, spec: OperatorSpec): + """Define the operator inputs and outputs.""" + spec.input(self._input_name_path).condition(ConditionType.NONE) + spec.output(self._output_name_image) + spec.output(self._output_name_filename).condition(ConditionType.NONE) + + def compute(self, op_input, op_output, context): + """Load the image file and emit it.""" + + # Try to get file path from input port + input_path = None + try: + input_path = op_input.receive(self._input_name_path) + except Exception: + pass + + # Validate input path or fall back to object attribute + if not input_path or not Path(input_path).is_file(): + self._logger.info(f"No or invalid file path from input port: {input_path}") + # Try to fall back to use the object attribute if it is valid + if self._input_path and self._input_path.is_file(): + input_path = self._input_path + else: + raise ValueError(f"No valid file path from input port or obj attribute: {self._input_path}") + + # Convert to Path object + image_path = Path(input_path) + + # Validate file extension + if image_path.suffix.lower() not in self.SUPPORTED_EXTENSIONS: + raise ValueError( + f"Unsupported file extension: {image_path.suffix}. " + f"Supported extensions: {self.SUPPORTED_EXTENSIONS}" + ) + + try: + # Load and process the image + image_obj = self._load_image(image_path) + + # Emit the image and filename + op_output.emit(image_obj, self._output_name_image) + op_output.emit(image_path.stem, self._output_name_filename) + + self._logger.info(f"Successfully loaded and emitted image: {image_path.name}") + + except Exception as e: + self._logger.error(f"Failed to load image {image_path}: {e}") + raise + + def _load_image(self, image_path: Path) -> Image: + """Load an image file and return as Image object.""" + # Load image using PIL + pil_image = PILImage.open(image_path) + + # Convert to RGB if necessary + if pil_image.mode != "RGB": + pil_image = pil_image.convert("RGB") + + # Convert to numpy array (HWC float32). Intensity scaling (to [0,1]) is typically handled by bundle. + image_array = np.array(pil_image).astype(np.float32) + + # Convert to channel-first when requested + if self._channel_first: + # PIL loads HWC; convert to CHW + image_array = np.transpose(image_array, (2, 0, 1)) + + # Create metadata + metadata = { + "filename": str(image_path), + "original_shape": image_array.shape, + "source_format": image_path.suffix.lower(), + } + + # Create Image object + return Image(image_array, metadata=metadata) + + +def test(): + """Test the ImageFileLoader operator.""" + import tempfile + + from PIL import Image as PILImageCreate + + # Create a temporary directory with a test image + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + + # Create test image + test_image_path = temp_path / "test_image.jpg" + img = PILImageCreate.new("RGB", (100, 100), color=(128, 64, 192)) + img.save(test_image_path) + + # Test the operator + fragment = Fragment() + loader = ImageFileLoader(fragment, input_path=test_image_path) + + # Simulate setup + from monai.deploy.core import OperatorSpec + + spec = OperatorSpec() + loader.setup(spec) + + # Simulate compute call + class MockInput: + def receive(self, name): + # Simulate no input from port, will fall back to object attribute + raise Exception("No input") + + class MockOutput: + def emit(self, data, name): + if name == "filename": + print(f"Emitted filename: {data}") + elif name == "image": + print(f"Emitted image with shape: {data.asnumpy().shape}") + + mock_input = MockInput() + mock_output = MockOutput() + + loader.compute(mock_input, mock_output, None) + + +if __name__ == "__main__": + test() diff --git a/tools/pipeline-generator/pipeline_generator/templates/operators/image_overlay_writer_operator.py b/tools/pipeline-generator/pipeline_generator/templates/operators/image_overlay_writer_operator.py new file mode 100644 index 00000000..a7d03913 --- /dev/null +++ b/tools/pipeline-generator/pipeline_generator/templates/operators/image_overlay_writer_operator.py @@ -0,0 +1,116 @@ +# Copyright 2025 MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from pathlib import Path +from typing import Optional, Tuple + +import numpy as np + +from monai.deploy.core import Fragment, Image, Operator, OperatorSpec +from monai.deploy.utils.importutil import optional_import + +PILImage, _ = optional_import("PIL", name="Image") + + +class ImageOverlayWriter(Operator): + """ + Image Overlay Writer + + Blends a segmentation mask onto an RGB image and saves the result as a PNG. + + Named inputs: + - image: original RGB frame as Image or ndarray (HWC, uint8/float) + - pred: predicted mask as Image or ndarray (H x W or 1 x H x W). If multi-channel + probability tensor is provided, you may pre-argmax before this operator. + - filename: base name (stem) for output file + """ + + def __init__( + self, + fragment: Fragment, + *args, + output_folder: Path, + alpha: float = 0.4, + color: Tuple[int, int, int] = (255, 0, 0), + threshold: Optional[float] = 0.5, + **kwargs, + ) -> None: + self._logger = logging.getLogger(f"{__name__}.{type(self).__name__}") + self._output_folder = Path(output_folder) + self._alpha = float(alpha) + self._color = tuple(int(c) for c in color) + self._threshold = threshold + super().__init__(fragment, *args, **kwargs) + + def setup(self, spec: OperatorSpec): + spec.input("image") + spec.input("pred") + spec.input("filename") + + def compute(self, op_input, op_output, context): + image_in = op_input.receive("image") + pred_in = op_input.receive("pred") + fname_stem = op_input.receive("filename") + + img = self._to_hwc_uint8(image_in) + mask = self._to_mask_uint8(pred_in) + + # Blend + overlay = self._blend_overlay(img, mask, self._alpha, self._color) + + self._output_folder.mkdir(parents=True, exist_ok=True) + out_path = self._output_folder / f"{fname_stem}_overlay.png" + PILImage.fromarray(overlay).save(out_path) + self._logger.info(f"Saved overlay PNG: {out_path}") + + def _to_hwc_uint8(self, image) -> np.ndarray: + if isinstance(image, Image): + arr: np.ndarray = image.asnumpy() + else: + arr = np.asarray(image) + if arr.ndim != 3 or arr.shape[2] not in (3, 4): + raise ValueError(f"Expected HWC image with 3 or 4 channels, got shape {arr.shape}") + # Drop alpha if present + if arr.shape[2] == 4: + arr = arr[..., :3] + # Scale/clip and convert + if not np.issubdtype(arr.dtype, np.uint8): + arr = np.clip(arr, 0, 255).astype(np.uint8) + return arr + + def _to_mask_uint8(self, pred) -> np.ndarray: + if isinstance(pred, Image): + arr: np.ndarray = pred.asnumpy() + else: + arr = np.asarray(pred) + arr = np.squeeze(arr) + if arr.ndim != 2: + raise ValueError(f"Expected 2D mask after squeeze, got shape {arr.shape}") + if self._threshold is not None and not np.issubdtype(arr.dtype, np.uint8): + arr = (arr > float(self._threshold)).astype(np.uint8) * 255 + elif arr.dtype != np.uint8: + # Assume already {0,1} + arr = (arr != 0).astype(np.uint8) * 255 + return arr + + @staticmethod + def _blend_overlay(img: np.ndarray, mask_u8: np.ndarray, alpha: float, color: Tuple[int, int, int]) -> np.ndarray: + # img: HWC uint8, mask_u8: HW uint8 + mask = (mask_u8 > 0).astype(np.float32)[..., None] + color_img = np.zeros_like(img, dtype=np.uint8) + color_img[..., 0] = color[0] + color_img[..., 1] = color[1] + color_img[..., 2] = color[2] + blended = ( + img.astype(np.float32) * (1.0 - alpha * mask) + color_img.astype(np.float32) * (alpha * mask) + ).astype(np.uint8) + return blended diff --git a/tools/pipeline-generator/pipeline_generator/templates/operators/json_results_writer_operator.py b/tools/pipeline-generator/pipeline_generator/templates/operators/json_results_writer_operator.py new file mode 100644 index 00000000..60847242 --- /dev/null +++ b/tools/pipeline-generator/pipeline_generator/templates/operators/json_results_writer_operator.py @@ -0,0 +1,234 @@ +# Copyright 2024 MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import logging +from pathlib import Path +from typing import Any, Dict, Union + +import numpy as np + +from monai.deploy.core import ConditionType, Fragment, Operator, OperatorSpec + + +class JSONResultsWriter(Operator): + """Write classification or prediction results to JSON files. + + This operator handles various types of model outputs (dictionaries, tensors, numpy arrays) + and saves them as JSON files with proper formatting. + + Named Inputs: + pred: Prediction results (dict, tensor, or numpy array) + filename: Optional filename for the output (without extension) + + File Output: + JSON files saved in the specified output folder + """ + + def __init__( + self, + fragment: Fragment, + *args, + output_folder: Union[str, Path], + result_key: str = "pred", + **kwargs, + ) -> None: + """Initialize the JSONResultsWriter. + + Args: + fragment: An instance of the Application class + output_folder: Path to folder for saving JSON results + result_key: Key to extract from prediction dict if applicable (default: "pred") + """ + self._logger = logging.getLogger("{}.{}".format(__name__, type(self).__name__)) + self.output_folder = Path(output_folder) + self.output_folder.mkdir(parents=True, exist_ok=True) + self.result_key = result_key + + super().__init__(fragment, *args, **kwargs) + + def setup(self, spec: OperatorSpec): + """Define the operator inputs.""" + spec.input("pred") + spec.input("filename").condition(ConditionType.NONE) # Optional input + + def compute(self, op_input, op_output, context): + """Process and save prediction results as JSON.""" + pred = op_input.receive("pred") + if pred is None: + self._logger.warning("No prediction received") + return + + # Try to get filename + filename = None + try: + filename = op_input.receive("filename") + except Exception: + pass + + if not filename: + # Generate a default filename + import datetime + + timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + filename = f"result_{timestamp}" + + # Process the prediction data + result_data = self._process_prediction(pred, filename) + + # Save as JSON + output_file = self.output_folder / f"{filename}_result.json" + with open(output_file, "w") as f: + json.dump(result_data, f, indent=2) + + self._logger.info(f"Saved results to {output_file}") + + # Print summary if it's a classification result + if "probabilities" in result_data: + self._print_classification_summary(result_data) + + def _process_prediction(self, pred: Any, filename: str) -> Dict[str, Any]: + """Process various prediction formats into a JSON-serializable dictionary.""" + result: Dict[str, Any] = {"filename": filename} + + # Handle dictionary predictions (e.g., from MonaiBundleInferenceOperator) + if isinstance(pred, dict): + if self.result_key in pred: + pred_data = pred[self.result_key] + else: + # Use the entire dict if our key isn't found + pred_data = pred + else: + pred_data = pred + + # Convert to numpy if it's a tensor + if hasattr(pred_data, "cpu"): # PyTorch tensor + pred_data = pred_data.cpu().numpy() + elif hasattr(pred_data, "asnumpy"): # MONAI MetaTensor + pred_data = pred_data.asnumpy() + + # Handle different prediction types + if isinstance(pred_data, np.ndarray): + if pred_data.ndim == 1: # 1D array (e.g., classification probabilities) + # Assume classification with probabilities + if len(pred_data) == 4: # Breast density classification + result["probabilities"] = { + "A": float(pred_data[0]), + "B": float(pred_data[1]), + "C": float(pred_data[2]), + "D": float(pred_data[3]), + } + else: + # Generic classification + result["probabilities"] = {f"class_{i}": float(pred_data[i]) for i in range(len(pred_data))} + + # Add predicted class + max_idx = int(np.argmax(pred_data)) + result["predicted_class"] = list(result["probabilities"].keys())[max_idx] + result["confidence"] = float(pred_data[max_idx]) + + elif pred_data.ndim == 2: # 2D array (batch of predictions) + # Take the first item if it's a batch + if pred_data.shape[0] == 1: + return self._process_prediction(pred_data[0], filename) + else: + # Multiple predictions + result["predictions"] = pred_data.tolist() + + else: + # Other array shapes - just convert to list + result["data"] = pred_data.tolist() + result["shape"] = list(pred_data.shape) + + elif isinstance(pred_data, (list, tuple)): + result["predictions"] = list(pred_data) + + elif isinstance(pred_data, dict): + # Already a dict, merge it + result.update(pred_data) + + else: + # Try to convert to string + result["prediction"] = str(pred_data) + + return result + + def _print_classification_summary(self, result: Dict[str, Any]): + """Print a summary of classification results.""" + print(f"\nClassification results for {result['filename']}:") + probs = result.get("probabilities", {}) + for class_name, prob in probs.items(): + print(f" {class_name}: {prob:.4f}") + if "predicted_class" in result: + print(f" Predicted: {result['predicted_class']} (confidence: {result['confidence']:.4f})") + + +def test(): + """Test the JSONResultsWriter operator.""" + import tempfile + + import numpy as np + + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + + # Test the operator + fragment = Fragment() + writer = JSONResultsWriter(fragment, output_folder=temp_path) + + # Simulate setup + from monai.deploy.core import OperatorSpec + + spec = OperatorSpec() + writer.setup(spec) + + # Test cases + class MockInput: + def __init__(self, pred, filename=None): + self.pred = pred + self.filename = filename + + def receive(self, name): + if name == "pred": + return self.pred + elif name == "filename": + if self.filename: + return self.filename + raise Exception("No filename") + + # Test 1: Classification probabilities + print("Test 1: Classification probabilities") + pred1 = {"pred": np.array([0.1, 0.7, 0.15, 0.05])} + mock_input1 = MockInput(pred1, "test_image_1") + writer.compute(mock_input1, None, None) + + # Test 2: Direct numpy array + print("\nTest 2: Direct numpy array") + pred2 = np.array([0.9, 0.05, 0.03, 0.02]) + mock_input2 = MockInput(pred2, "test_image_2") + writer.compute(mock_input2, None, None) + + # Test 3: No filename provided + print("\nTest 3: No filename provided") + pred3 = {"classification": [0.2, 0.8]} + mock_input3 = MockInput(pred3) + writer.compute(mock_input3, None, None) + + # List generated files + print("\nGenerated files:") + for json_file in temp_path.glob("*.json"): + print(f" {json_file.name}") + with open(json_file) as f: + print(f" Content: {json.load(f)}") + + +if __name__ == "__main__": + test() diff --git a/tools/pipeline-generator/pipeline_generator/templates/operators/llama3_vila_inference_operator.py b/tools/pipeline-generator/pipeline_generator/templates/operators/llama3_vila_inference_operator.py new file mode 100644 index 00000000..0f00f8fc --- /dev/null +++ b/tools/pipeline-generator/pipeline_generator/templates/operators/llama3_vila_inference_operator.py @@ -0,0 +1,320 @@ +# Copyright 2025 MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from pathlib import Path +from typing import Any, Dict, Optional, Union + +import numpy as np +import torch + +from monai.deploy.core import AppContext, Fragment, Image, Operator, OperatorSpec +from monai.deploy.utils.importutil import optional_import + +# Lazy imports for transformers +AutoConfig, _ = optional_import("transformers", name="AutoConfig") +AutoModelForCausalLM, _ = optional_import("transformers", name="AutoModelForCausalLM") +AutoTokenizer, _ = optional_import("transformers", name="AutoTokenizer") + +PILImage, _ = optional_import("PIL", name="Image") +ImageDraw, _ = optional_import("PIL.ImageDraw") +ImageFont, _ = optional_import("PIL.ImageFont") + + +class Llama3VILAInferenceOperator(Operator): + """Inference operator for Llama3-VILA-M3-3B vision-language model. + + This operator takes an image and text prompt as input and generates + text and/or image outputs based on the model's response and the + specified output type. + + The operator supports three output types: + - json: Returns the model's text response as JSON data + - image: Returns the original image (placeholder for future image generation) + - image_overlay: Returns the image with text overlay + + Inputs: + image: Image object to analyze + prompt: Text prompt for the model + output_type: Expected output type (json, image, or image_overlay) + request_id: Unique identifier for the request + generation_params: Dictionary of generation parameters + + Outputs: + result: The generated result (format depends on output_type) + output_type: The output type (passed through) + request_id: The request ID (passed through) + """ + + def __init__( + self, + fragment: Fragment, + *args, + app_context: AppContext, + model_path: Union[str, Path], + device: Optional[str] = None, + **kwargs, + ) -> None: + """Initialize the Llama3VILAInferenceOperator. + + Args: + fragment: An instance of the Application class + app_context: Application context + model_path: Path to the Llama3-VILA model directory + device: Device to run inference on (default: auto-detect) + """ + self._logger = logging.getLogger("{}.{}".format(__name__, type(self).__name__)) + self.app_context = app_context + self.model_path = Path(model_path) + + # Auto-detect device if not specified + if device is None: + self.device = "cuda" if torch.cuda.is_available() else "cpu" + else: + self.device = device + + self._logger.info(f"Using device: {self.device}") + + super().__init__(fragment, *args, **kwargs) + + # Model components will be loaded during setup + self.model = None + self.tokenizer = None + self.image_processor = None + + def setup(self, spec: OperatorSpec): + """Define the operator inputs and outputs.""" + # Inputs + spec.input("image") + spec.input("prompt") + spec.input("output_type") + spec.input("request_id") + spec.input("generation_params") + + # Outputs + spec.output("result") + spec.output("output_type") + spec.output("request_id") + + # Load the model during setup + self._load_model() + + def _load_model(self): + """Load the Llama3-VILA model and its components.""" + try: + self._logger.info(f"Loading model from {self.model_path}") + + # Load model configuration + config = AutoConfig.from_pretrained(self.model_path) + + # Load tokenizer + self.tokenizer = AutoTokenizer.from_pretrained(self.model_path / "llm", use_fast=False) + + # For LLaVA-style models, we typically need to handle image processing + # and model loading in a specific way. For now, we'll create a simplified + # inference pipeline that demonstrates the structure. + + # Note: In a production implementation, you would load the actual model here + # using the appropriate LLaVA/VILA loading mechanism + self._logger.info("Model components loaded successfully") + + # Set a flag to indicate we're using a mock implementation + self._mock_mode = True + self._logger.warning( + "Running in mock mode - actual model loading requires VILA/LLaVA dependencies. " + "Results will be simulated based on output type." + ) + + except Exception as e: + self._logger.error(f"Failed to load model: {e}") + self._mock_mode = True + + def _preprocess_image(self, image: Image) -> torch.Tensor: + """Preprocess the image for model input.""" + # Get the numpy array from the Image object + image_array = image.asnumpy() + + # Ensure HWC format + if image_array.ndim == 3 and image_array.shape[0] <= 4: # Likely CHW + image_array = np.transpose(image_array, (1, 2, 0)) + + # Normalize to [0, 1] if needed + if image_array.max() > 1.0: + image_array = image_array / 255.0 + + # In a real implementation, you would use the model's image processor + # For now, we'll just convert to tensor + return torch.from_numpy(image_array).float() + + def _generate_response(self, image_tensor: torch.Tensor, prompt: str, generation_params: Dict[str, Any]) -> str: + """Generate text response from the model.""" + if self._mock_mode: + # Mock response based on common medical VQA patterns + mock_responses = { + "what is this image showing": "This medical image shows anatomical structures with various tissue densities and contrast patterns.", # noqa: B950 + "summarize key findings": "Key findings include: 1) Normal anatomical structures visible, 2) No obvious pathological changes detected, 3) Image quality is adequate for assessment.", # noqa: B950 + "is there a focal lesion": "No focal lesion is identified in the visible field of view.", # noqa: B950 + "describe the image": "This appears to be a medical imaging study showing cross-sectional anatomy with good tissue contrast.", # noqa: B950 + } + + # Find best matching response + prompt_lower = prompt.lower() + for key, response in mock_responses.items(): + if key in prompt_lower: + return response + + # Default response + return f"Analysis of the medical image based on the prompt: {prompt!r}. [Mock response - actual model not loaded]" + + # In a real implementation, you would: + # 1. Tokenize the prompt + # 2. Prepare the image features + # 3. Run the model + # 4. Decode the output + return "Model inference not implemented" + + def _create_json_result( + self, + text_response: str, + request_id: str, + prompt: Optional[str] = None, + image_metadata: Optional[Dict] = None, + ) -> Dict[str, Any]: + """Create a JSON result from the text response.""" + result = { + "request_id": request_id, + "response": text_response, + "status": "success", + } + if prompt: + result["prompt"] = prompt + if image_metadata and "filename" in image_metadata: + result["image"] = image_metadata["filename"] + return result + + def _create_image_overlay(self, image: Image, text: str) -> Image: + """Create an image with text overlay.""" + # Get the numpy array + image_array = image.asnumpy() + + # Ensure HWC format and uint8 + if image_array.ndim == 3 and image_array.shape[0] <= 4: # Likely CHW + image_array = np.transpose(image_array, (1, 2, 0)) + + if image_array.max() <= 1.0: + image_array = (image_array * 255).astype(np.uint8) + else: + image_array = image_array.astype(np.uint8) + + # Convert to PIL Image + pil_image = PILImage.fromarray(image_array) + + # Create a drawing context + draw = ImageDraw.Draw(pil_image) + + # Add text overlay + # Break text into lines for better display + words = text.split() + lines = [] + current_line: list[str] = [] + max_width = pil_image.width - 20 # Leave margin + + # Simple text wrapping (in production, use proper text metrics) + chars_per_line = max_width // 10 # Rough estimate + current_length = 0 + + for word in words: + if current_length + len(word) + 1 > chars_per_line: + lines.append(" ".join(current_line)) + current_line = [word] + current_length = len(word) + else: + current_line.append(word) + current_length += len(word) + 1 + + if current_line: + lines.append(" ".join(current_line)) + + # Draw text with background + y_offset = 10 + for line in lines[:5]: # Limit to 5 lines + # Draw background rectangle + bbox = [10, y_offset, max_width + 10, y_offset + 20] + draw.rectangle(bbox, fill=(0, 0, 0, 180)) + + # Draw text + draw.text((15, y_offset + 2), line, fill=(255, 255, 255)) + y_offset += 25 + + # Convert back to numpy array + result_array = np.array(pil_image).astype(np.float32) + + # Create new Image object + metadata = image.metadata().copy() if image.metadata() else {} + metadata["overlay_text"] = text + + return Image(result_array, metadata=metadata) + + def compute(self, op_input, op_output, context): + """Run inference and generate results.""" + # Get inputs + image = op_input.receive("image") + prompt = op_input.receive("prompt") + output_type = op_input.receive("output_type") + request_id = op_input.receive("request_id") + generation_params = op_input.receive("generation_params") + + self._logger.info(f"Processing request {request_id} with output type {output_type!r}") + + try: + # Preprocess image + image_tensor = self._preprocess_image(image) + + # Generate text response + text_response = self._generate_response(image_tensor, prompt, generation_params) + + # Get image metadata if available + image_metadata = image.metadata() if hasattr(image, "metadata") and callable(image.metadata) else None + + # Create result based on output type + if output_type == "json": + result = self._create_json_result(text_response, request_id, prompt, image_metadata) + elif output_type == "image": + # For now, just return the original image + # In future, this could generate new images + result = image + elif output_type == "image_overlay": + result = self._create_image_overlay(image, text_response) + else: + self._logger.warning(f"Unknown output type: {output_type}, defaulting to json") + result = self._create_json_result(text_response, request_id, prompt, image_metadata) + + # Emit outputs + op_output.emit(result, "result") + op_output.emit(output_type, "output_type") + op_output.emit(request_id, "request_id") + + self._logger.info(f"Successfully processed request {request_id}") + + except Exception as e: + self._logger.error(f"Error processing request {request_id}: {e}") + + # Emit error result + error_result = { + "request_id": request_id, + "prompt": prompt, + "error": str(e), + "status": "error", + } + op_output.emit(error_result, "result") + op_output.emit(output_type, "output_type") + op_output.emit(request_id, "request_id") + raise e from None diff --git a/tools/pipeline-generator/pipeline_generator/templates/operators/monai_classification_operator.py b/tools/pipeline-generator/pipeline_generator/templates/operators/monai_classification_operator.py new file mode 100644 index 00000000..bd8ca9b2 --- /dev/null +++ b/tools/pipeline-generator/pipeline_generator/templates/operators/monai_classification_operator.py @@ -0,0 +1,263 @@ +# Copyright 2024 MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from pathlib import Path +from typing import List, Optional, Union + +import torch + +from monai.bundle import ConfigParser +from monai.deploy.core import AppContext, Fragment, Image, Operator, OperatorSpec +from monai.deploy.utils.importutil import optional_import +from monai.transforms import Compose + +# Dynamic class imports to match MONAI model loader behavior +monai, _ = optional_import("monai") +torchvision, _ = optional_import("torchvision") + +globals_dict = { + "torch": torch, + "monai": monai, + "torchvision": torchvision, +} + + +class MonaiClassificationOperator(Operator): + """Operator for MONAI classification models that use Python model definitions. + + This operator handles models like TorchVisionFCModel that require: + 1. Loading a Python class definition + 2. Instantiating the model + 3. Loading state dict weights + + It supports models from MONAI bundles that don't use TorchScript. + """ + + DEFAULT_PRE_PROC_CONFIG = ["preprocessing", "transforms"] + DEFAULT_POST_PROC_CONFIG = ["postprocessing", "transforms"] + + def __init__( + self, + fragment: Fragment, + *args, + app_context: AppContext, + bundle_path: Union[str, Path], + config_names: Optional[Union[List[str], str]] = None, + **kwargs, + ): + """Initialize the operator. + + Args: + fragment: Fragment instance + app_context: Application context + bundle_path: Path to the MONAI bundle + config_names: Names of configs to use + """ + self._logger = logging.getLogger("{}.{}".format(__name__, type(self).__name__)) + self._executing = False + + # Set attributes before calling super().__init__ since setup() is called from there + self.app_context = app_context + self.bundle_path = Path(bundle_path) + self.config_names = config_names or [] + + super().__init__(fragment, *args, **kwargs) + + # Will be loaded during setup + self._model = None + self._pre_processor = None + self._post_processor = None + self._inference_config = None + + def setup(self, spec: OperatorSpec): + """Set up the operator.""" + spec.input("image") + spec.output("pred") + + def _load_bundle(self): + """Load the MONAI bundle configuration and model.""" + # Load inference config + inference_path = self.bundle_path / "configs" / "inference.json" + if not inference_path.exists(): + raise FileNotFoundError(f"Inference config not found: {inference_path}") + + self._logger.info(f"Loading inference config from: {inference_path}") + parser = ConfigParser() + parser.read_config(str(inference_path)) + + # Set up global imports for dynamic loading + parser.globals = globals_dict + + # Store raw config for later use + self._inference_config = parser.config + + # Load preprocessing - get the transforms directly + if "preprocessing" in parser.config and "transforms" in parser.config["preprocessing"]: + pre_transforms = parser.get_parsed_content("preprocessing#transforms") + # Skip LoadImaged since our image is already loaded + filtered_transforms = [] + for t in pre_transforms: + if type(t).__name__ not in ["LoadImaged", "LoadImage"]: + filtered_transforms.append(t) + else: + self._logger.info(f"Skipping {type(t).__name__} transform as image is already loaded") + if filtered_transforms: + self._pre_processor = Compose(filtered_transforms) + self._logger.info(f"Loaded preprocessing transforms: {[type(t).__name__ for t in filtered_transforms]}") + + # Load model + self._load_model(parser) + + # Load postprocessing - get the transforms directly + if "postprocessing" in parser.config and "transforms" in parser.config["postprocessing"]: + post_transforms = parser.get_parsed_content("postprocessing#transforms") + self._post_processor = Compose(post_transforms) + self._logger.info(f"Loaded postprocessing transforms: {[type(t).__name__ for t in post_transforms]}") + + def _load_model(self, parser: ConfigParser): + """Load the model from the bundle.""" + # Get model definition - parse it to instantiate the model + try: + model = parser.get_parsed_content("network_def") + if model is None: + raise ValueError("Failed to parse network_def") + self._logger.info(f"Loaded model: {type(model).__name__}") + except Exception as e: + self._logger.error(f"Error loading model definition: {e}") + raise + + # Load model weights + model_path = self.bundle_path / "models" / "model.pt" + if not model_path.exists(): + # Try alternative paths + alt_paths = [ + self.bundle_path / "models" / "model.pth", + self.bundle_path / "model.pt", + self.bundle_path / "model.pth", + ] + for alt_path in alt_paths: + if alt_path.exists(): + model_path = alt_path + break + else: + raise FileNotFoundError(f"Model file not found. Looked in: {model_path} and alternatives") + + self._logger.info(f"Loading model weights from: {model_path}") + + # Detect device + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + # Load state dict + # Use weights_only=True for security (requires PyTorch 1.13+) + try: + state_dict = torch.load(str(model_path), map_location=device, weights_only=True) + except TypeError: + self._logger.warning("Using torch.load without weights_only restriction - ensure model files are trusted") + state_dict = torch.load(str(model_path), map_location=device) + + # Handle different state dict formats + if "state_dict" in state_dict: + state_dict = state_dict["state_dict"] + elif "model" in state_dict: + state_dict = state_dict["model"] + + # Load weights into model + model.load_state_dict(state_dict) + model = model.to(device) + model.eval() + + self._model = model + self._device = device + self._logger.info(f"Model loaded successfully on device: {device}") + + def compute(self, op_input, op_output, context): + """Run inference on the input image.""" + input_image = op_input.receive("image") + if input_image is None: + raise ValueError("No input image received") + + # Ensure we're not processing multiple times + if self._executing: + self._logger.warning("Already executing, skipping") + return + + self._executing = True + + try: + # Lazy load model if not already loaded + if self._model is None: + self._logger.info("Loading model on first compute call") + self._load_bundle() + + # Convert Image to tensor format expected by MONAI + if isinstance(input_image, Image): + # Image data is already in CHW format from ImageFileLoader + image_tensor = torch.from_numpy(input_image.asnumpy()).float() + else: + image_tensor = input_image + + self._logger.info(f"Input tensor shape: {image_tensor.shape}") + + # Move to device first + image_tensor = image_tensor.to(self._device) + + # Apply preprocessing + if self._pre_processor: + # MONAI dict transforms expect dict format with key "image" + # Since all our transforms end with 'd', we need dict format + data = {"image": image_tensor} + data = self._pre_processor(data) + image_tensor = data["image"] + self._logger.info(f"After preprocessing shape: {image_tensor.shape}") + + # Add batch dimension if needed (after preprocessing) + if image_tensor.dim() == 3: + image_tensor = image_tensor.unsqueeze(0) + + # Run inference + with torch.no_grad(): + pred = self._model(image_tensor) + + # Apply postprocessing + if self._post_processor: + data = {"pred": pred} + data = self._post_processor(data) + pred = data["pred"] + + # Convert to dict format for output + if isinstance(pred, torch.Tensor): + # For classification, output is typically probabilities per class + pred_dict = {} + if pred.dim() == 2 and pred.shape[0] == 1: + # Single batch, multiple classes + pred = pred.squeeze(0) + + # Create dict with class probabilities + for i, prob in enumerate(pred.cpu().numpy()): + pred_dict[f"class_{i}"] = float(prob) + + # Add predicted class + pred_dict["predicted_class"] = int(torch.argmax(pred).item()) + + result = pred_dict + else: + result = pred + + # Emit the result + op_output.emit(result, "pred") + self._logger.info(f"Inference completed. Result: {result}") + + except Exception as e: + self._logger.error(f"Error during inference: {e}") + raise + finally: + self._executing = False diff --git a/tools/pipeline-generator/pipeline_generator/templates/operators/nifti_writer_operator.py b/tools/pipeline-generator/pipeline_generator/templates/operators/nifti_writer_operator.py new file mode 100644 index 00000000..96334b0f --- /dev/null +++ b/tools/pipeline-generator/pipeline_generator/templates/operators/nifti_writer_operator.py @@ -0,0 +1,139 @@ +# Copyright 2024 MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import logging +from pathlib import Path + +import numpy as np + +from monai.deploy.core import ConditionType, Fragment, Operator, OperatorSpec +from monai.deploy.core.domain import Image +from monai.deploy.utils.importutil import optional_import + +nibabel, _ = optional_import("nibabel") + + +class NiftiWriter(Operator): + """ + This operator writes segmentation results to NIfTI files. + + Named input: + image: Image data to save (Image object or numpy array) + filename: Optional filename to use for saving + + Named output: + None + """ + + def __init__( + self, + fragment: Fragment, + *args, + output_folder: Path, + output_postfix: str = "seg", + output_extension: str = ".nii.gz", + **kwargs, + ) -> None: + """Creates an instance of the NIfTI writer. + + Args: + fragment (Fragment): An instance of the Application class which is derived from Fragment. + output_folder (Path): Path to output folder. + output_postfix (str): Postfix to add to output filenames. Defaults to "seg". + output_extension (str): File extension for output files. Defaults to ".nii.gz". + """ + self._logger = logging.getLogger("{}.{}".format(__name__, type(self).__name__)) + self.output_folder = Path(output_folder) + self.output_postfix = output_postfix + self.output_extension = output_extension + + # Input names + self.input_name_image = "image" + self.input_name_filename = "filename" + + super().__init__(fragment, *args, **kwargs) + + def setup(self, spec: OperatorSpec): + spec.input(self.input_name_image) + spec.input(self.input_name_filename).condition(ConditionType.NONE) # Optional + + def compute(self, op_input, op_output, context): + """Save the image to a NIfTI file.""" + + # Get inputs + image = op_input.receive(self.input_name_image) + + # Try to get filename + filename = None + try: + filename = op_input.receive(self.input_name_filename) + except Exception: + pass + + if image is None: + return + + # Get the image array + if isinstance(image, Image): + image_array = image.asnumpy() if hasattr(image, "asnumpy") else np.array(image) + # Try to get metadata + metadata = ( + image.metadata() if callable(image.metadata) else image.metadata if hasattr(image, "metadata") else {} + ) + else: + image_array = np.array(image) + metadata = {} + + # Remove batch dimension if present + if image_array.ndim == 4 and image_array.shape[0] == 1: + image_array = image_array[0] + + # Remove channel dimension if it's 1 + if image_array.ndim == 4 and image_array.shape[-1] == 1: + image_array = image_array[..., 0] + + # Use filename or generate one + if not filename: + filename = "output" + + # Create output path + self.output_folder.mkdir(parents=True, exist_ok=True) + + # Generate output filename + # Handle template variables in output_postfix (e.g., "@output_postfix") + if self.output_postfix and self.output_postfix.startswith("@"): + # Default to "trans" for template variables + actual_postfix = "trans" + else: + actual_postfix = self.output_postfix + + if actual_postfix: + output_filename = f"{filename}_{actual_postfix}{self.output_extension}" + else: + output_filename = f"{filename}{self.output_extension}" + + output_path = self.output_folder / output_filename + + # Get affine matrix from metadata if available + affine = np.eye(4) + if isinstance(metadata, dict) and "affine" in metadata: + affine = np.array(metadata["affine"]) + + # Transpose from (N, H, W) to (H, W, N) for NIfTI format + if image_array.ndim == 3: + image_array = np.transpose(image_array, [1, 2, 0]) + + # Save as NIfTI + nifti_img = nibabel.Nifti1Image(image_array.astype(np.float32), affine) + nibabel.save(nifti_img, str(output_path)) + + self._logger.info(f"Saved segmentation to: {output_path}") diff --git a/tools/pipeline-generator/pipeline_generator/templates/operators/prompts_loader_operator.py b/tools/pipeline-generator/pipeline_generator/templates/operators/prompts_loader_operator.py new file mode 100644 index 00000000..257bd9ce --- /dev/null +++ b/tools/pipeline-generator/pipeline_generator/templates/operators/prompts_loader_operator.py @@ -0,0 +1,203 @@ +# Copyright 2025 MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import uuid +from pathlib import Path +from typing import Any, Dict, List, Optional + +import numpy as np +import yaml # type: ignore + +from monai.deploy.core import Fragment, Image, Operator, OperatorSpec +from monai.deploy.utils.importutil import optional_import + +PILImage, _ = optional_import("PIL", name="Image") + + +class PromptsLoaderOperator(Operator): + """Load prompts from a YAML file and emit them one at a time with associated images. + + This operator reads a prompts.yaml file with the following format: + + ```yaml + defaults: + max_new_tokens: 256 + temperature: 0.2 + top_p: 0.9 + prompts: + - prompt: Summarize key findings. + image: img1.png + output: json + - prompt: Is there a focal lesion? + image: img2.png + output: image + max_new_tokens: 128 + ``` + + For each prompt, it emits: + - image: The loaded image as an Image object + - prompt: The prompt text + - output_type: The expected output type (json, image, or image_overlay) + - request_id: A unique identifier for the request + - generation_params: A dictionary of generation parameters + + The operator processes prompts sequentially and stops execution when all prompts + have been processed. + """ + + def __init__( + self, + fragment: Fragment, + *args, + input_folder: Path, + **kwargs, + ) -> None: + """Initialize the PromptsLoaderOperator. + + Args: + fragment: An instance of the Application class + input_folder: Path to folder containing prompts.yaml and image files + """ + self._logger = logging.getLogger("{}.{}".format(__name__, type(self).__name__)) + self._input_folder = Path(input_folder) + + super().__init__(fragment, *args, **kwargs) + + def setup(self, spec: OperatorSpec): + """Define the operator outputs.""" + spec.output("image") + spec.output("prompt") + spec.output("output_type") + spec.output("request_id") + spec.output("generation_params") + + # Load and parse the prompts file + self._prompts_data = self._load_prompts() + self._current_index = 0 + + if not self._prompts_data: + self._logger.warning(f"No prompts found in {self._input_folder}/prompts.yaml") + else: + self._logger.info(f"Found {len(self._prompts_data)} prompts to process") + + def _load_prompts(self) -> List[Dict[str, Any]]: + """Load and parse the prompts.yaml file.""" + prompts_file = self._input_folder / "prompts.yaml" + + if not prompts_file.exists(): + self._logger.error(f"prompts.yaml not found in {self._input_folder}") + return [] + + try: + with open(prompts_file, "r") as f: + data = yaml.safe_load(f) + + defaults = data.get("defaults", {}) + prompts = data.get("prompts", []) + + # Merge defaults with each prompt + processed_prompts = [] + for prompt in prompts: + # Create generation parameters by merging defaults with prompt-specific params + gen_params = defaults.copy() + + # Override with prompt-specific parameters + for key in ["max_new_tokens", "temperature", "top_p"]: + if key in prompt: + gen_params[key] = prompt[key] + + processed_prompts.append( + { + "prompt": prompt.get("prompt", ""), + "image": prompt.get("image", ""), + "output_type": prompt.get("output", "json"), + "generation_params": gen_params, + } + ) + + return processed_prompts + + except Exception as e: + self._logger.error(f"Error loading prompts.yaml: {e}") + return [] + + def _load_image(self, image_filename: str) -> Optional[Image]: + """Load an image file and convert it to an Image object.""" + image_path = self._input_folder / image_filename + + if not image_path.exists(): + self._logger.error(f"Image file not found: {image_path}") + return None + + try: + # Load image using PIL + pil_image = PILImage.open(image_path) + + # Convert to RGB if necessary + if pil_image.mode != "RGB": + pil_image = pil_image.convert("RGB") + + # Convert to numpy array (HWC format, float32) + # Note: For VLM models, we typically keep HWC format + image_array = np.array(pil_image).astype(np.float32) + + # Create metadata + metadata = { + "filename": str(image_path), + "original_shape": image_array.shape, + "source_format": image_path.suffix.lower(), + } + + # Create Image object + return Image(image_array, metadata=metadata) + + except Exception as e: + self._logger.error(f"Failed to load image {image_path}: {e}") + return None + + def compute(self, op_input, op_output, context): + """Process one prompt and emit it.""" + + # Check if we have more prompts to process + if self._current_index >= len(self._prompts_data): + # No more prompts to process + self._logger.info("All prompts have been processed") + self.fragment.stop_execution() + return + + # Get the current prompt data + prompt_data = self._prompts_data[self._current_index] + + # Load the associated image + image = self._load_image(prompt_data["image"]) + if image is None: + self._logger.error("Skipping prompt due to image load failure") + self._current_index += 1 + return + + # Generate a unique request ID + request_id = str(uuid.uuid4()) + + # Emit all the data + op_output.emit(image, "image") + op_output.emit(prompt_data["prompt"], "prompt") + op_output.emit(prompt_data["output_type"], "output_type") + op_output.emit(request_id, "request_id") + op_output.emit(prompt_data["generation_params"], "generation_params") + + self._logger.info( + f"Emitted prompt {self._current_index + 1}/{len(self._prompts_data)}: " + f"'{prompt_data['prompt'][:50]}...' with image {prompt_data['image']}" + ) + + # Move to the next prompt + self._current_index += 1 diff --git a/tools/pipeline-generator/pipeline_generator/templates/operators/vlm_results_writer_operator.py b/tools/pipeline-generator/pipeline_generator/templates/operators/vlm_results_writer_operator.py new file mode 100644 index 00000000..6401d408 --- /dev/null +++ b/tools/pipeline-generator/pipeline_generator/templates/operators/vlm_results_writer_operator.py @@ -0,0 +1,167 @@ +# Copyright 2025 MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import logging +from pathlib import Path +from typing import Any, Dict + +import numpy as np + +from monai.deploy.core import Fragment, Image, Operator, OperatorSpec +from monai.deploy.utils.importutil import optional_import + +PILImage, _ = optional_import("PIL", name="Image") + + +class VLMResultsWriterOperator(Operator): + """Write vision-language model results to disk based on output type. + + This operator receives results from the VLM inference operator and writes + them to the output directory in the appropriate format: + + - json: Writes the result as a JSON file named {request_id}.json + - image: Writes the image as a PNG file named {request_id}.png + - image_overlay: Writes the image with overlay as a PNG file named {request_id}_overlay.png + + The operator handles results sequentially and writes each one to disk as it's received. + + Inputs: + result: The generated result (format depends on output_type) + output_type: The output type (json, image, or image_overlay) + request_id: The request ID used for naming output files + """ + + def __init__( + self, + fragment: Fragment, + *args, + output_folder: Path, + **kwargs, + ) -> None: + """Initialize the VLMResultsWriterOperator. + + Args: + fragment: An instance of the Application class + output_folder: Path to folder where results will be written + """ + self._logger = logging.getLogger("{}.{}".format(__name__, type(self).__name__)) + self._output_folder = Path(output_folder) + + # Create output directory if it doesn't exist + self._output_folder.mkdir(parents=True, exist_ok=True) + + super().__init__(fragment, *args, **kwargs) + + # Track number of results written + self._results_written = 0 + + def setup(self, spec: OperatorSpec): + """Define the operator inputs.""" + spec.input("result") + spec.input("output_type") + spec.input("request_id") + + def _write_json_result(self, result: Dict[str, Any], request_id: str): + """Write JSON result to disk.""" + output_path = self._output_folder / f"{request_id}.json" + + try: + with open(output_path, "w") as f: + json.dump(result, f, indent=2) + self._logger.info(f"Wrote JSON result to {output_path}") + except Exception as e: + self._logger.error(f"Failed to write JSON result: {e}") + + def _write_image_result(self, image: Image, request_id: str, suffix: str = ""): + """Write image result to disk.""" + output_filename = f"{request_id}{suffix}.png" + output_path = self._output_folder / output_filename + + try: + # Get numpy array from Image object + image_array = image.asnumpy() + + # Ensure HWC format + if image_array.ndim == 3 and image_array.shape[0] <= 4: # Likely CHW + image_array = np.transpose(image_array, (1, 2, 0)) + + # Convert to uint8 if needed + if image_array.dtype == np.float32 or image_array.dtype == np.float64: + if image_array.max() <= 1.0: + image_array = (image_array * 255).astype(np.uint8) + else: + image_array = image_array.astype(np.uint8) + + # Save using PIL + pil_image = PILImage.fromarray(image_array) + pil_image.save(output_path) + + self._logger.info(f"Wrote image result to {output_path}") + + except Exception as e: + self._logger.error(f"Failed to write image result: {e}") + + def compute(self, op_input, op_output, context): + """Write results to disk based on output type.""" + # Receive inputs + result = op_input.receive("result") + output_type = op_input.receive("output_type") + request_id = op_input.receive("request_id") + + self._logger.info(f"Writing result for request {request_id} with output type {output_type!r}") + + try: + if output_type == "json": + if isinstance(result, dict): + self._write_json_result(result, request_id) + else: + # Convert to dict if needed + self._write_json_result({"result": str(result)}, request_id) + + elif output_type == "image": + if isinstance(result, Image): + self._write_image_result(result, request_id) + else: + self._logger.error(f"Expected Image object for image output, got {type(result)}") + + elif output_type == "image_overlay": + if isinstance(result, Image): + self._write_image_result(result, request_id, suffix="_overlay") + else: + self._logger.error(f"Expected Image object for image_overlay output, got {type(result)}") + + else: + self._logger.warning(f"Unknown output type: {output_type}") + # Write as JSON fallback + self._write_json_result({"result": str(result), "output_type": output_type}, request_id) + + self._results_written += 1 + self._logger.info(f"Total results written: {self._results_written}") + + except Exception as e: + self._logger.error(f"Error writing result for request {request_id}: {e}") + + # Try to write error file + error_path = self._output_folder / f"{request_id}_error.json" + try: + with open(error_path, "w") as f: + json.dump( + { + "request_id": request_id, + "error": str(e), + "output_type": output_type, + }, + f, + indent=2, + ) + except Exception: + pass diff --git a/tools/pipeline-generator/pipeline_generator/templates/requirements.txt.j2 b/tools/pipeline-generator/pipeline_generator/templates/requirements.txt.j2 new file mode 100644 index 00000000..92c33f87 --- /dev/null +++ b/tools/pipeline-generator/pipeline_generator/templates/requirements.txt.j2 @@ -0,0 +1,34 @@ +# Requirements for {{ app_title }} +# Generated from model: {{ model_id }} + +# MONAI Deploy App SDK and dependencies +monai-deploy-app-sdk>=3.0.0 + + +# Required by MONAI Deploy SDK (always needed) +pydicom>=2.3.0 # Required by MONAI Deploy SDK even for NIfTI apps +highdicom>=0.18.2 # Required for DICOM segmentation support + +{% if input_type == "image" %} +# Image loading dependencies +Pillow>=8.0.0 # For loading JPEG/PNG images +{% elif input_type == "nifti" or output_type == "nifti" %} +SimpleITK>=2.0.2 + +{% endif %} + + +# Any additional requirements specified in bundle metadata +{% if required_packages_version %} +{% for pkg, ver in required_packages_version.items() %} +{{ pkg }}=={{ ver }} +{% endfor %} + +{% endif %} + +# Additional dependencies specified in generator config +{% if extra_dependencies %} +{% for dep in extra_dependencies %} +{{ dep }} +{% endfor %} +{% endif %} diff --git a/tools/pipeline-generator/pyproject.toml b/tools/pipeline-generator/pyproject.toml new file mode 100644 index 00000000..b63a7716 --- /dev/null +++ b/tools/pipeline-generator/pyproject.toml @@ -0,0 +1,58 @@ +# Copyright 2025 MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[project] +name = "pipeline-generator" +version = "1.0.0" +description = "A CLI tool for generating MONAI Deploy pipelines from MONAI Bundles" +readme = "README.md" +requires-python = ">=3.10,<3.11" +authors = [{ name = "MONAI" }] +dependencies = [ + "click>=8.2.1", + "pyyaml>=6.0.2", + "huggingface-hub>=0.34.3", + "pydantic>=2.11.7", + "rich>=14.1.0", + "jinja2>=3.1.6", +] + +[project.scripts] +pg = "pipeline_generator.cli.main:cli" + +[build-system] +requires = ["hatchling>=1.25.0"] +build-backend = "hatchling.build" + +[dependency-groups] +dev = [ + "pytest>=8.4.1", + "pytest-cov>=6.2.1", + "black>=25.1.0", + "flake8>=7.3.0", + "mypy>=1.17.1", + "types-pyyaml>=6.0.12.20250516", +] + +[tool.black] +line-length = 100 +target-version = ['py310'] + +[tool.mypy] +python_version = "3.10" +warn_return_any = true +warn_unused_configs = true +disallow_untyped_defs = true +check_untyped_defs = true +exclude = ["tests/"] + +[tool.flake8] +max-line-length = 100 diff --git a/tools/pipeline-generator/test.sh b/tools/pipeline-generator/test.sh new file mode 100755 index 00000000..21f03ca1 --- /dev/null +++ b/tools/pipeline-generator/test.sh @@ -0,0 +1,53 @@ +#! /bin/bash + +# List all available pipelines +uv run pg list + +rm -r results* test_* +uv run pg gen MONAI/breast_density_classification --output test_breast_den_cls +uv run pg run test_breast_den_cls --input test_breast_den_cls/model/sample_data/A/ --output ./results3 +uv run pg run test_breast_den_cls --input test_breast_den_cls/model/sample_data/B/ --output ./results3 +uv run pg run test_breast_den_cls --input test_breast_den_cls/model/sample_data/C/ --output ./results3 + +rm -r results* test_* +uv run pg gen MONAI/multi_organ_segmentation --output test_multiorgan_seg +uv run pg run test_multiorgan_seg/ --input /home/vicchang/Downloads/Task09_Spleen/Task09_Spleen/imagesTs --output ./results2 + +rm -r results* test_* +uv run pg gen MONAI/spleen_ct_segmentation --output test_spleen_ct_seg +uv run pg run test_spleen_ct_seg/ --input /home/vicchang/Downloads/Task09_Spleen/Task09_Spleen/imagesTs --output ./results + +rm -r results* test_* +uv run pg gen MONAI/endoscopic_tool_segmentation --output test_endo_tool_seg +uv run pg run test_endo_tool_seg/ --input /home/vicchang/Downloads/instrument_5_8_testing/instrument_dataset_5/left_frames --output ./results + +rm -r results* test_* +uv run pg gen MONAI/wholeBrainSeg_Large_UNEST_segmentation --output test_whole_brainseg_large +uv run pg run test_whole_brainseg_large/ --input /home/vicchang/Downloads/Task01_BrainTumour/imagesTs --output ./results + +rm -r results* test_* +uv run pg gen MONAI/wholeBody_ct_segmentation --output test_wholeBody_ct_segmentation +uv run pg run test_wholeBody_ct_segmentation/ --input /home/vicchang/Downloads/Task09_Spleen/Task09_Spleen/imagesTs --output ./results + +rm -r results* test_* +uv run pg gen MONAI/swin_unetr_btcv_segmentation --output test_swin_unetr_btcv_segmentation +uv run pg run test_swin_unetr_btcv_segmentation --input /home/vicchang/Downloads/Task09_Spleen/Task09_Spleen/imagesTs --output ./results + +rm -r results* test_* +uv run pg gen MONAI/Llama3-VILA-M3-3B --output test_llama3 +uv run pg run test_llama3 --input /home/vicchang/sc/github/monai/monai-deploy-app-sdk/tools/test_inputs --output ./results + +rm -r results* test_* +uv run pg gen MONAI/Llama3-VILA-M3-8B --output test_llama3_8b +uv run pg run test_llama3_8b --input /home/vicchang/sc/github/monai/monai-deploy-app-sdk/tools/test_inputs --output ./results + +rm -r results* test_* +uv run pg gen MONAI/Llama3-VILA-M3-13B --output test_llama3_13b +uv run pg run test_llama3_13b --input /home/vicchang/sc/github/monai/monai-deploy-app-sdk/tools/test_inputs --output ./results + + +rm -r results* test_* +uv run pg gen MONAI/retinalOCT_RPD_segmentation --output test_retinal_oct_seg +uv run pg run test_retinal_oct_seg --input /home/vicchang/sc/github/monai/monai-deploy-app-sdk/tools/pipeline-generator/test_retinal_oct_seg/model/sample_data --output ./results + + diff --git a/tools/pipeline-generator/tests/test_bundle_downloader.py b/tools/pipeline-generator/tests/test_bundle_downloader.py new file mode 100644 index 00000000..e918b655 --- /dev/null +++ b/tools/pipeline-generator/tests/test_bundle_downloader.py @@ -0,0 +1,472 @@ +# Copyright 2025 MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for bundle downloader.""" + +import json +from unittest.mock import patch + +import pytest +from pipeline_generator.generator.bundle_downloader import BundleDownloader + + +class TestBundleDownloader: + """Test bundle downloader functionality.""" + + def setup_method(self): + """Set up test fixtures.""" + self.downloader = BundleDownloader() + + @patch("pipeline_generator.generator.bundle_downloader.snapshot_download") + def test_download_bundle_success(self, mock_snapshot_download, tmp_path): + """Test successful bundle download.""" + output_dir = tmp_path / "output" + cache_dir = tmp_path / "cache" + + # Mock successful download + mock_snapshot_download.return_value = str(output_dir / "model") + + result = self.downloader.download_bundle("MONAI/spleen_ct_segmentation", output_dir, cache_dir) + + assert result == output_dir / "model" + mock_snapshot_download.assert_called_once_with( + repo_id="MONAI/spleen_ct_segmentation", + local_dir=output_dir / "model", + cache_dir=cache_dir, + ) + + @patch("pipeline_generator.generator.bundle_downloader.snapshot_download") + def test_download_bundle_failure(self, mock_snapshot_download, tmp_path): + """Test bundle download failure.""" + output_dir = tmp_path / "output" + + # Mock download failure + mock_snapshot_download.side_effect = Exception("Download failed") + + with pytest.raises(Exception, match="Download failed"): + self.downloader.download_bundle("MONAI/nonexistent", output_dir) + + def test_get_bundle_metadata_from_configs(self, tmp_path): + """Test getting bundle metadata from configs directory.""" + bundle_path = tmp_path / "bundle" + configs_dir = bundle_path / "configs" + configs_dir.mkdir(parents=True) + + # Create metadata.json + metadata = { + "name": "Test Model", + "version": "1.0.0", + "description": "Test description", + } + metadata_file = configs_dir / "metadata.json" + metadata_file.write_text(json.dumps(metadata)) + + result = self.downloader.get_bundle_metadata(bundle_path) + + assert result is not None + assert result["name"] == "Test Model" + assert result["version"] == "1.0.0" + + def test_get_bundle_metadata_from_root(self, tmp_path): + """Test getting bundle metadata from root directory.""" + bundle_path = tmp_path / "bundle" + bundle_path.mkdir() + + # Create metadata.json in root + metadata = {"name": "Test Model", "version": "1.0.0"} + metadata_file = bundle_path / "metadata.json" + metadata_file.write_text(json.dumps(metadata)) + + result = self.downloader.get_bundle_metadata(bundle_path) + + assert result is not None + assert result["name"] == "Test Model" + + def test_get_bundle_metadata_not_found(self, tmp_path): + """Test getting bundle metadata when file doesn't exist.""" + bundle_path = tmp_path / "bundle" + bundle_path.mkdir() + + result = self.downloader.get_bundle_metadata(bundle_path) + + assert result is None + + def test_get_bundle_metadata_invalid_json(self, tmp_path): + """Test getting bundle metadata with invalid JSON.""" + bundle_path = tmp_path / "bundle" + configs_dir = bundle_path / "configs" + configs_dir.mkdir(parents=True) + + # Create invalid metadata.json + metadata_file = configs_dir / "metadata.json" + metadata_file.write_text("invalid json") + + result = self.downloader.get_bundle_metadata(bundle_path) + + assert result is None + + def test_get_inference_config_success(self, tmp_path): + """Test getting inference configuration.""" + bundle_path = tmp_path / "bundle" + configs_dir = bundle_path / "configs" + configs_dir.mkdir(parents=True) + + # Create inference.json + inference_config = { + "preprocessing": {"transforms": [{"name": "LoadImaged"}, {"name": "EnsureChannelFirstd"}]}, + "postprocessing": {"transforms": [{"name": "Activationsd", "sigmoid": True}]}, + } + inference_file = configs_dir / "inference.json" + inference_file.write_text(json.dumps(inference_config)) + + result = self.downloader.get_inference_config(bundle_path) + + assert result is not None + assert "preprocessing" in result + assert len(result["preprocessing"]["transforms"]) == 2 + + def test_get_inference_config_not_found(self, tmp_path): + """Test getting inference config when file doesn't exist.""" + bundle_path = tmp_path / "bundle" + bundle_path.mkdir() + + result = self.downloader.get_inference_config(bundle_path) + + assert result is None + + def test_detect_model_file_torchscript(self, tmp_path): + """Test detecting TorchScript model file.""" + bundle_path = tmp_path / "bundle" + models_dir = bundle_path / "models" + models_dir.mkdir(parents=True) + + # Create model.ts file + model_file = models_dir / "model.ts" + model_file.write_text("torchscript model") + + result = self.downloader.detect_model_file(bundle_path) + + assert result == models_dir / "model.ts" + + def test_detect_model_file_pytorch(self, tmp_path): + """Test detecting PyTorch model file.""" + bundle_path = tmp_path / "bundle" + models_dir = bundle_path / "models" + models_dir.mkdir(parents=True) + + # Create model.pt file + model_file = models_dir / "model.pt" + model_file.write_bytes(b"pytorch model") + + result = self.downloader.detect_model_file(bundle_path) + + assert result == models_dir / "model.pt" + + def test_detect_model_file_onnx(self, tmp_path): + """Test detecting ONNX model file.""" + bundle_path = tmp_path / "bundle" + models_dir = bundle_path / "models" + models_dir.mkdir(parents=True) + + # Create model.onnx file + model_file = models_dir / "model.onnx" + model_file.write_bytes(b"onnx model") + + result = self.downloader.detect_model_file(bundle_path) + + assert result == models_dir / "model.onnx" + + def test_detect_model_file_non_standard_location(self, tmp_path): + """Test detecting model file in non-standard location.""" + bundle_path = tmp_path / "bundle" + custom_dir = bundle_path / "custom" / "location" + custom_dir.mkdir(parents=True) + + # Create model.pt file in custom location + model_file = custom_dir / "model.pt" + model_file.write_bytes(b"pytorch model") + + result = self.downloader.detect_model_file(bundle_path) + + assert result == custom_dir / "model.pt" + + def test_detect_model_file_in_root(self, tmp_path): + """Test detecting model file in root directory.""" + bundle_path = tmp_path / "bundle" + bundle_path.mkdir() + + # Create model.pt in root + model_file = bundle_path / "model.pt" + model_file.write_bytes(b"pytorch model") + + result = self.downloader.detect_model_file(bundle_path) + + assert result == bundle_path / "model.pt" + + def test_detect_model_file_not_found(self, tmp_path): + """Test detecting model file when none exists.""" + bundle_path = tmp_path / "bundle" + bundle_path.mkdir() + + result = self.downloader.detect_model_file(bundle_path) + + assert result is None + + def test_organize_bundle_structure_flat_to_structured(self, tmp_path): + """Test organizing flat bundle structure into standard format.""" + bundle_path = tmp_path / "bundle" + bundle_path.mkdir() + + # Create files in flat structure + metadata_file = bundle_path / "metadata.json" + inference_file = bundle_path / "inference.json" + model_pt_file = bundle_path / "model.pt" + model_ts_file = bundle_path / "model.ts" + + metadata_file.write_text('{"name": "Test"}') + inference_file.write_text('{"config": "test"}') + model_pt_file.touch() + model_ts_file.touch() + + # Organize structure + self.downloader.organize_bundle_structure(bundle_path) + + # Check that files were moved to proper locations + assert (bundle_path / "configs" / "metadata.json").exists() + assert (bundle_path / "configs" / "inference.json").exists() + assert (bundle_path / "models" / "model.pt").exists() + assert (bundle_path / "models" / "model.ts").exists() + + # Check that original files were moved (not copied) + assert not metadata_file.exists() + assert not inference_file.exists() + assert not model_pt_file.exists() + assert not model_ts_file.exists() + + def test_organize_bundle_structure_already_structured(self, tmp_path): + """Test organizing bundle that already has proper structure.""" + bundle_path = tmp_path / "bundle" + configs_dir = bundle_path / "configs" + models_dir = bundle_path / "models" + configs_dir.mkdir(parents=True) + models_dir.mkdir(parents=True) + + # Create files in proper structure + metadata_file = configs_dir / "metadata.json" + model_file = models_dir / "model.pt" + metadata_file.write_text('{"name": "Test"}') + model_file.touch() + + # Should not change anything + self.downloader.organize_bundle_structure(bundle_path) + + # Files should remain in place + assert metadata_file.exists() + assert model_file.exists() + + def test_organize_bundle_structure_partial_structure(self, tmp_path): + """Test organizing bundle with partial structure.""" + bundle_path = tmp_path / "bundle" + configs_dir = bundle_path / "configs" + configs_dir.mkdir(parents=True) + + # Create metadata in configs but model in root + metadata_file = configs_dir / "metadata.json" + model_file = bundle_path / "model.pt" + metadata_file.write_text('{"name": "Test"}') + model_file.touch() + + # Organize structure + self.downloader.organize_bundle_structure(bundle_path) + + # Metadata should stay, model should move + assert metadata_file.exists() + assert (bundle_path / "models" / "model.pt").exists() + assert not model_file.exists() + + def test_detect_model_file_multiple_models(self, tmp_path): + """Test detecting model file with multiple model files (returns first found).""" + bundle_path = tmp_path / "bundle" + models_dir = bundle_path / "models" + models_dir.mkdir(parents=True) + + # Create multiple model files + (models_dir / "model.ts").write_text("torchscript") + (models_dir / "model.pt").write_bytes(b"pytorch") + (models_dir / "model.onnx").write_bytes(b"onnx") + + result = self.downloader.detect_model_file(bundle_path) + + # Should return the first one found (model.ts in this case) + assert result == models_dir / "model.ts" + + @patch("pipeline_generator.generator.bundle_downloader.logger") + def test_get_bundle_metadata_logs_error(self, mock_logger, tmp_path): + """Test that metadata reading errors are logged.""" + bundle_path = tmp_path / "bundle" + configs_dir = bundle_path / "configs" + configs_dir.mkdir(parents=True) + + # Create a file that will cause a read error + metadata_file = configs_dir / "metadata.json" + metadata_file.write_text("invalid json") + + result = self.downloader.get_bundle_metadata(bundle_path) + + assert result is None + mock_logger.error.assert_called() + + @patch("pipeline_generator.generator.bundle_downloader.logger") + def test_get_inference_config_logs_error(self, mock_logger, tmp_path): + """Test that inference config reading errors are logged.""" + bundle_path = tmp_path / "bundle" + configs_dir = bundle_path / "configs" + configs_dir.mkdir(parents=True) + + # Create a file that will cause a read error + inference_file = configs_dir / "inference.json" + inference_file.write_text("invalid json") + + result = self.downloader.get_inference_config(bundle_path) + + assert result is None + mock_logger.error.assert_called() + + def test_organize_bundle_structure_subdirectory_models(self, tmp_path): + """Test organizing models from subdirectories to main models/ directory.""" + bundle_path = tmp_path / "bundle" + models_dir = bundle_path / "models" + subdir = models_dir / "A100" + subdir.mkdir(parents=True) + + # Create model file in subdirectory + subdir_model = subdir / "dynunet_FT_trt_16.ts" + subdir_model.write_text("tensorrt model") + + # Organize structure + self.downloader.organize_bundle_structure(bundle_path) + + # Model should be moved to main models/ directory with standard name + assert (models_dir / "model.ts").exists() + assert not subdir_model.exists() + assert not subdir.exists() # Empty subdirectory should be removed + + def test_organize_bundle_structure_prefers_pytorch_over_tensorrt(self, tmp_path): + """Test that PyTorch models are preferred over TensorRT models.""" + bundle_path = tmp_path / "bundle" + models_dir = bundle_path / "models" + subdir = models_dir / "A100" + subdir.mkdir(parents=True) + + # Create both PyTorch and TensorRT models in subdirectory + pytorch_model = subdir / "dynunet_FT.pt" + tensorrt_model = subdir / "dynunet_FT_trt_16.ts" + pytorch_model.write_bytes(b"pytorch model") + tensorrt_model.write_text("tensorrt model") + + # Organize structure + self.downloader.organize_bundle_structure(bundle_path) + + # PyTorch model should be preferred and moved + assert (models_dir / "model.pt").exists() + assert not (models_dir / "model.ts").exists() + assert not pytorch_model.exists() + # TensorRT model should remain in subdirectory + assert tensorrt_model.exists() + + def test_organize_bundle_structure_standard_naming_pytorch(self, tmp_path): + """Test renaming PyTorch models to standard names.""" + bundle_path = tmp_path / "bundle" + models_dir = bundle_path / "models" + models_dir.mkdir(parents=True) + + # Create PyTorch model with custom name + custom_model = models_dir / "dynunet_FT.pt" + custom_model.write_bytes(b"pytorch model") + + # Organize structure + self.downloader.organize_bundle_structure(bundle_path) + + # Model should be renamed to standard name + assert (models_dir / "model.pt").exists() + assert not custom_model.exists() + + def test_organize_bundle_structure_standard_naming_torchscript(self, tmp_path): + """Test renaming TorchScript models to standard names when no PyTorch model exists.""" + bundle_path = tmp_path / "bundle" + models_dir = bundle_path / "models" + models_dir.mkdir(parents=True) + + # Create only TorchScript model with custom name + custom_model = models_dir / "custom_model.ts" + custom_model.write_text("torchscript model") + + # Organize structure + self.downloader.organize_bundle_structure(bundle_path) + + # Model should be renamed to standard name + assert (models_dir / "model.ts").exists() + assert not custom_model.exists() + + def test_organize_bundle_structure_skips_when_suitable_model_exists(self, tmp_path): + """Test that subdirectory organization is skipped when suitable model already exists.""" + bundle_path = tmp_path / "bundle" + models_dir = bundle_path / "models" + subdir = models_dir / "A100" + subdir.mkdir(parents=True) + + # Create model in main directory + main_model = models_dir / "existing_model.pt" + main_model.write_bytes(b"existing pytorch model") + + # Create model in subdirectory + subdir_model = subdir / "dynunet_FT_trt_16.ts" + subdir_model.write_text("tensorrt model") + + # Organize structure + self.downloader.organize_bundle_structure(bundle_path) + + # Main model should be renamed to standard name + assert (models_dir / "model.pt").exists() + assert not main_model.exists() + + # Subdirectory model should remain untouched + assert subdir_model.exists() + assert subdir.exists() + + def test_organize_bundle_structure_multiple_extensions_preference(self, tmp_path): + """Test extension preference order: .pt > .onnx > .ts.""" + bundle_path = tmp_path / "bundle" + models_dir = bundle_path / "models" + subdir = models_dir / "A100" + subdir.mkdir(parents=True) + + # Create models with different extensions in subdirectory + pt_model = subdir / "model.pt" + onnx_model = subdir / "model.onnx" + ts_model = subdir / "model.ts" + + pt_model.write_bytes(b"pytorch model") + onnx_model.write_bytes(b"onnx model") + ts_model.write_text("torchscript model") + + # Organize structure + self.downloader.organize_bundle_structure(bundle_path) + + # Should prefer .pt model + assert (models_dir / "model.pt").exists() + assert not (models_dir / "model.onnx").exists() + assert not (models_dir / "model.ts").exists() + assert not pt_model.exists() + + # Other models should remain in subdirectory + assert onnx_model.exists() + assert ts_model.exists() diff --git a/tools/pipeline-generator/tests/test_cli.py b/tools/pipeline-generator/tests/test_cli.py new file mode 100644 index 00000000..7354f1bd --- /dev/null +++ b/tools/pipeline-generator/tests/test_cli.py @@ -0,0 +1,282 @@ +# Copyright 2025 MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for CLI commands.""" + +from unittest.mock import Mock, patch + +from click.testing import CliRunner +from pipeline_generator.cli.main import cli +from pipeline_generator.core.models import ModelInfo + + +class TestCLI: + """Test CLI commands.""" + + def setup_method(self): + """Set up test fixtures.""" + self.runner = CliRunner() + + def test_cli_help(self): + """Test CLI help command.""" + result = self.runner.invoke(cli, ["--help"]) + assert result.exit_code == 0 + assert "Pipeline Generator" in result.output + assert "Generate MONAI Deploy and Holoscan pipelines" in result.output + + def test_cli_version(self): + """Test CLI version command.""" + result = self.runner.invoke(cli, ["--version"]) + assert result.exit_code == 0 + assert "version" in result.output.lower() + + @patch("pipeline_generator.cli.main.HuggingFaceClient") + @patch("pipeline_generator.cli.main.load_config") + def test_list_command_table_format(self, mock_load_config, mock_client_class): + """Test list command with table format.""" + # Mock the configuration + mock_settings = Mock() + mock_settings.get_all_endpoints.return_value = [Mock(organization="MONAI")] + mock_settings.endpoints = [] # Add empty endpoints list + mock_load_config.return_value = mock_settings + + # Mock the HuggingFace client + mock_client = Mock() + mock_client_class.return_value = mock_client + + # Mock model data + test_models = [ + ModelInfo( + model_id="MONAI/test_model1", + name="Test Model 1", + downloads=100, + likes=10, + is_monai_bundle=True, + ), + ModelInfo( + model_id="MONAI/test_model2", + name="Test Model 2", + downloads=200, + likes=20, + is_monai_bundle=False, + ), + ] + mock_client.list_models_from_endpoints.return_value = test_models + + # Run command + result = self.runner.invoke(cli, ["list"]) + + assert result.exit_code == 0 + assert "Fetching models from HuggingFace" in result.output + assert "MONAI/test_model1" in result.output + assert "MONAI/test_model2" in result.output + assert "Total models: 2" in result.output + assert "MONAI Bundles: 1" in result.output + + @patch("pipeline_generator.cli.main.HuggingFaceClient") + @patch("pipeline_generator.cli.main.load_config") + def test_list_command_bundles_only(self, mock_load_config, mock_client_class): + """Test list command with bundles-only filter.""" + # Mock setup + mock_settings = Mock() + mock_settings.get_all_endpoints.return_value = [Mock(organization="MONAI")] + mock_settings.endpoints = [] # Add empty endpoints list + mock_load_config.return_value = mock_settings + + mock_client = Mock() + mock_client_class.return_value = mock_client + + # Mock model data with mixed bundle status + test_models = [ + ModelInfo(model_id="MONAI/bundle1", name="Bundle 1", is_monai_bundle=True), + ModelInfo(model_id="MONAI/model1", name="Model 1", is_monai_bundle=False), + ModelInfo(model_id="MONAI/bundle2", name="Bundle 2", is_monai_bundle=True), + ] + mock_client.list_models_from_endpoints.return_value = test_models + + # Run command with bundles-only filter + result = self.runner.invoke(cli, ["list", "--bundles-only"]) + + assert result.exit_code == 0 + assert "MONAI/bundle1" in result.output + assert "MONAI/bundle2" in result.output + assert "MONAI/model1" not in result.output + assert "Total models: 2" in result.output # Only bundles shown + + @patch("pipeline_generator.cli.main.HuggingFaceClient") + @patch("pipeline_generator.cli.main.load_config") + def test_list_command_simple_format(self, mock_load_config, mock_client_class): + """Test list command with simple format.""" + # Mock setup + mock_settings = Mock() + mock_settings.get_all_endpoints.return_value = [Mock(organization="MONAI")] + mock_settings.endpoints = [] # Add empty endpoints list + mock_load_config.return_value = mock_settings + + mock_client = Mock() + mock_client_class.return_value = mock_client + + test_models = [ModelInfo(model_id="MONAI/test", name="Test", is_monai_bundle=True)] + mock_client.list_models_from_endpoints.return_value = test_models + + # Run command with simple format + result = self.runner.invoke(cli, ["list", "--format", "simple"]) + + assert result.exit_code == 0 + assert "📦 MONAI/test" in result.output + + def test_list_command_with_config(self): + """Test list command with custom config file.""" + with self.runner.isolated_filesystem(): + # Create a test config file + with open("test_config.yaml", "w") as f: + f.write( + """ +endpoints: + - organization: "TestOrg" + description: "Test organization" +""" + ) + + # Run command with config file + with patch("pipeline_generator.cli.main.HuggingFaceClient") as mock_client_class: + mock_client = Mock() + mock_client_class.return_value = mock_client + mock_client.list_models_from_endpoints.return_value = [] + + result = self.runner.invoke(cli, ["--config", "test_config.yaml", "list"]) + + assert result.exit_code == 0 + + @patch("pipeline_generator.cli.main.HuggingFaceClient") + @patch("pipeline_generator.cli.main.load_config") + def test_list_command_json_format(self, mock_load_config, mock_client_class): + """Test list command with JSON format output.""" + import json + + # Mock setup + mock_settings = Mock() + mock_settings.endpoints = [] + mock_load_config.return_value = mock_settings + + mock_client = Mock() + mock_client_class.return_value = mock_client + + test_models = [ + ModelInfo( + model_id="MONAI/test", + name="Test Model", + is_monai_bundle=True, + downloads=100, + likes=10, + tags=["medical", "segmentation"], + ) + ] + mock_client.list_models_from_endpoints.return_value = test_models + + # Run command with JSON format + result = self.runner.invoke(cli, ["list", "--format", "json"]) + + assert result.exit_code == 0 + + # Extract JSON from output (skip header line) + lines = result.output.strip().split("\n") + json_start = -1 + for i, line in enumerate(lines): + if line.strip().startswith("["): + json_start = i + break + + if json_start >= 0: + json_text = "\n".join(lines[json_start:]) + if "\nTotal models:" in json_text: + json_text = json_text[: json_text.rfind("\nTotal models:")] + + data = json.loads(json_text) + assert len(data) == 1 + assert data[0]["model_id"] == "MONAI/test" + assert data[0]["is_monai_bundle"] is True + + @patch("pipeline_generator.cli.main.HuggingFaceClient") + @patch("pipeline_generator.cli.main.load_config") + def test_list_command_no_models(self, mock_load_config, mock_client_class): + """Test list command when no models are found.""" + # Mock setup + mock_settings = Mock() + mock_settings.endpoints = [] + mock_load_config.return_value = mock_settings + + mock_client = Mock() + mock_client_class.return_value = mock_client + mock_client.list_models_from_endpoints.return_value = [] + + result = self.runner.invoke(cli, ["list"]) + + assert result.exit_code == 0 + assert "No models found" in result.output or "Total models: 0" in result.output + + @patch("pipeline_generator.cli.main.HuggingFaceClient") + @patch("pipeline_generator.cli.main.load_config") + def test_list_command_tested_only(self, mock_load_config, mock_client_class): + """Test list command with tested-only filter.""" + # Mock setup + mock_settings = Mock() + + # Create tested models in settings + tested_model = Mock() + tested_model.model_id = "MONAI/tested_model" + + mock_endpoint = Mock() + mock_endpoint.models = [tested_model] + mock_settings.endpoints = [mock_endpoint] + + mock_load_config.return_value = mock_settings + + mock_client = Mock() + mock_client_class.return_value = mock_client + + # Mock the list response + test_models = [ + ModelInfo(model_id="MONAI/tested_model", name="Tested Model", is_monai_bundle=True), + ModelInfo( + model_id="MONAI/untested_model", + name="Untested Model", + is_monai_bundle=True, + ), + ] + mock_client.list_models_from_endpoints.return_value = test_models + + # Test with tested-only filter + result = self.runner.invoke(cli, ["list", "--tested-only"]) + + assert result.exit_code == 0 + assert "MONAI/tested_model" in result.output + assert "MONAI/untested_model" not in result.output + + @patch("pipeline_generator.cli.main.AppGenerator") + @patch("pipeline_generator.cli.main.load_config") + def test_gen_command_error_handling(self, mock_load_config, mock_generator_class): + """Test gen command error handling.""" + mock_settings = Mock() + mock_load_config.return_value = mock_settings + + mock_generator = Mock() + mock_generator_class.return_value = mock_generator + + # Make generate_app raise an exception + mock_generator.generate_app.side_effect = Exception("Test error") + + with patch("pipeline_generator.cli.main.logger") as mock_logger: + result = self.runner.invoke(cli, ["gen", "MONAI/test_model"]) + + # Should log the exception + assert mock_logger.exception.called + assert result.exit_code != 0 diff --git a/tools/pipeline-generator/tests/test_gen_command.py b/tools/pipeline-generator/tests/test_gen_command.py new file mode 100644 index 00000000..34a43f0e --- /dev/null +++ b/tools/pipeline-generator/tests/test_gen_command.py @@ -0,0 +1,211 @@ +# Copyright 2025 MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for the gen command.""" + +from pathlib import Path +from unittest.mock import Mock, patch + +from click.testing import CliRunner +from pipeline_generator.cli.main import cli + + +class TestGenCommand: + """Test the gen command functionality.""" + + def setup_method(self): + """Set up test fixtures.""" + self.runner = CliRunner() + + @patch("pipeline_generator.cli.main.AppGenerator") + def test_gen_command_success(self, mock_generator_class, tmp_path): + """Test successful application generation.""" + # Mock the generator + mock_generator = Mock() + mock_generator_class.return_value = mock_generator + mock_generator.generate_app.return_value = tmp_path / "output" + + with self.runner.isolated_filesystem(): + result = self.runner.invoke(cli, ["gen", "MONAI/spleen_ct_segmentation"]) + + assert result.exit_code == 0 + assert "Generating MONAI Deploy application" in result.output + assert "✓ Application generated successfully!" in result.output + mock_generator.generate_app.assert_called_once() + + @patch("pipeline_generator.cli.main.AppGenerator") + def test_gen_command_with_custom_output(self, mock_generator_class, tmp_path): + """Test gen command with custom output directory.""" + mock_generator = Mock() + mock_generator_class.return_value = mock_generator + mock_generator.generate_app.return_value = tmp_path / "custom_output" + + with self.runner.isolated_filesystem(): + result = self.runner.invoke( + cli, + ["gen", "MONAI/spleen_ct_segmentation", "--output", "custom_output"], + ) + + assert result.exit_code == 0 + assert "Output directory: custom_output" in result.output + + # Verify the generator was called with correct parameters + call_args = mock_generator.generate_app.call_args + assert call_args[1]["output_dir"] == Path("custom_output") + + @patch("pipeline_generator.cli.main.AppGenerator") + def test_gen_command_with_app_name(self, mock_generator_class, tmp_path): + """Test gen command with custom app name.""" + mock_generator = Mock() + mock_generator_class.return_value = mock_generator + mock_generator.generate_app.return_value = tmp_path / "output" + + with self.runner.isolated_filesystem(): + result = self.runner.invoke( + cli, + ["gen", "MONAI/spleen_ct_segmentation", "--app-name", "MyCustomApp"], + ) + + assert result.exit_code == 0 + + # Verify the generator was called with custom app name + call_args = mock_generator.generate_app.call_args + assert call_args[1]["app_name"] == "MyCustomApp" + + @patch("pipeline_generator.cli.main.AppGenerator") + def test_gen_command_with_format(self, mock_generator_class, tmp_path): + """Test gen command with specific format.""" + mock_generator = Mock() + mock_generator_class.return_value = mock_generator + mock_generator.generate_app.return_value = tmp_path / "output" + + with self.runner.isolated_filesystem(): + result = self.runner.invoke(cli, ["gen", "MONAI/spleen_ct_segmentation", "--format", "nifti"]) + + assert result.exit_code == 0 + assert "Format: nifti" in result.output + + # Verify the generator was called with format + call_args = mock_generator.generate_app.call_args + assert call_args[1]["data_format"] == "nifti" + + def test_gen_command_existing_directory_without_force(self): + """Test gen command when output directory exists without force.""" + with self.runner.isolated_filesystem(): + # Create existing output directory with a file + output_dir = Path("output") + output_dir.mkdir() + (output_dir / "existing_file.txt").write_text("test") + + result = self.runner.invoke(cli, ["gen", "MONAI/spleen_ct_segmentation"]) + + assert result.exit_code == 1 + assert "Error: Output directory" in result.output + assert "already exists" in result.output + + @patch("pipeline_generator.cli.main.AppGenerator") + def test_gen_command_existing_directory_with_force(self, mock_generator_class, tmp_path): + """Test gen command when output directory exists with force.""" + mock_generator = Mock() + mock_generator_class.return_value = mock_generator + mock_generator.generate_app.return_value = tmp_path / "output" + + with self.runner.isolated_filesystem(): + # Create existing output directory + output_dir = Path("output") + output_dir.mkdir() + (output_dir / "existing_file.txt").write_text("test") + + result = self.runner.invoke(cli, ["gen", "MONAI/spleen_ct_segmentation", "--force"]) + + assert result.exit_code == 0 + assert "✓ Application generated successfully!" in result.output + + @patch("pipeline_generator.cli.main.AppGenerator") + def test_gen_command_bundle_download_error(self, mock_generator_class): + """Test gen command when bundle download fails.""" + mock_generator = Mock() + mock_generator_class.return_value = mock_generator + mock_generator.generate_app.side_effect = RuntimeError("Failed to download bundle") + + with self.runner.isolated_filesystem(): + result = self.runner.invoke(cli, ["gen", "MONAI/nonexistent_model"]) + + assert result.exit_code == 1 + assert "Error generating application" in result.output + + @patch("pipeline_generator.cli.main.AppGenerator") + def test_gen_command_generation_error(self, mock_generator_class): + """Test gen command when generation fails.""" + mock_generator = Mock() + mock_generator_class.return_value = mock_generator + mock_generator.generate_app.side_effect = Exception("Generation failed") + + with self.runner.isolated_filesystem(): + result = self.runner.invoke(cli, ["gen", "MONAI/spleen_ct_segmentation"]) + + assert result.exit_code == 1 + assert "Error generating application" in result.output + + @patch("pipeline_generator.cli.main.AppGenerator") + def test_gen_command_shows_generated_files(self, mock_generator_class): + """Test that gen command shows list of generated files.""" + + with self.runner.isolated_filesystem(): + # Create output directory with files + output_dir = Path("output") + output_dir.mkdir() + (output_dir / "app.py").write_text("# app") + (output_dir / "requirements.txt").write_text("monai") + (output_dir / "README.md").write_text("# README") + model_dir = output_dir / "model" + model_dir.mkdir() + (model_dir / "model.pt").write_text("model") + + # Mock the generator to return our prepared directory + mock_generator = Mock() + mock_generator_class.return_value = mock_generator + mock_generator.generate_app.return_value = output_dir + + result = self.runner.invoke( + cli, + [ + "gen", + "MONAI/spleen_ct_segmentation", + "--force", + ], # Use force since dir exists + ) + + assert result.exit_code == 0 + assert "Generated files:" in result.output + assert "• app.py" in result.output + assert "• requirements.txt" in result.output + assert "• README.md" in result.output + assert "• model/model.pt" in result.output + + @patch("pipeline_generator.cli.main.AppGenerator") + def test_gen_command_shows_next_steps(self, mock_generator_class, tmp_path): + """Test that gen command shows next steps.""" + mock_generator = Mock() + mock_generator_class.return_value = mock_generator + mock_generator.generate_app.return_value = tmp_path / "output" + + with self.runner.isolated_filesystem(): + result = self.runner.invoke(cli, ["gen", "MONAI/spleen_ct_segmentation"]) + + assert result.exit_code == 0 + assert "Next steps:" in result.output + assert "Option 1: Run with uv (recommended)" in result.output + assert "Option 2: Run with pg directly" in result.output + assert "pg run output" in result.output + assert "Option 3: Run manually" in result.output + assert "cd output" in result.output + assert "pip install -r requirements.txt" in result.output diff --git a/tools/pipeline-generator/tests/test_generator.py b/tools/pipeline-generator/tests/test_generator.py new file mode 100644 index 00000000..25eeea86 --- /dev/null +++ b/tools/pipeline-generator/tests/test_generator.py @@ -0,0 +1,1120 @@ +# Copyright 2025 MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for the app generator.""" + +import tempfile +from pathlib import Path +from unittest.mock import patch + +import pytest +from pipeline_generator.generator import AppGenerator, BundleDownloader + + +class TestBundleDownloader: + """Test BundleDownloader class.""" + + def test_init(self): + """Test BundleDownloader initialization.""" + downloader = BundleDownloader() + assert downloader.api is not None + + @patch("pipeline_generator.generator.bundle_downloader.snapshot_download") + def test_download_bundle(self, mock_snapshot_download): + """Test downloading a bundle.""" + downloader = BundleDownloader() + + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + mock_snapshot_download.return_value = str(temp_path / "model") + + result = downloader.download_bundle("MONAI/test_model", temp_path) + + assert result == temp_path / "model" + mock_snapshot_download.assert_called_once() + + def test_get_bundle_metadata(self): + """Test reading bundle metadata.""" + downloader = BundleDownloader() + + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + + # Create test metadata + metadata_path = temp_path / "configs" / "metadata.json" + metadata_path.parent.mkdir(parents=True) + metadata_path.write_text('{"name": "Test Model", "version": "1.0"}') + + metadata = downloader.get_bundle_metadata(temp_path) + + assert metadata is not None + assert metadata["name"] == "Test Model" + assert metadata["version"] == "1.0" + + def test_detect_model_file(self): + """Test detecting model file in bundle.""" + downloader = BundleDownloader() + + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + + # Create test model file + models_dir = temp_path / "models" + models_dir.mkdir() + model_file = models_dir / "model.ts" + model_file.touch() + + detected = downloader.detect_model_file(temp_path) + + assert detected is not None + assert detected.name == "model.ts" + + +class TestAppGenerator: + """Test AppGenerator class.""" + + def test_init(self): + """Test AppGenerator initialization.""" + generator = AppGenerator() + assert generator.downloader is not None + assert generator.env is not None + + def test_extract_organ_name(self): + """Test organ name extraction.""" + generator = AppGenerator() + + # Test with known organ names + assert generator._extract_organ_name("spleen_ct_segmentation", {}) == "Spleen" + assert generator._extract_organ_name("liver_tumor_seg", {}) == "Liver" + assert generator._extract_organ_name("kidney_segmentation", {}) == "Kidney" + + # Test with metadata + assert generator._extract_organ_name("test_model", {"organ": "Heart"}) == "Heart" + + # Test default + assert generator._extract_organ_name("unknown_model", {}) == "Organ" + + def test_prepare_context(self): + """Test context preparation for templates.""" + generator = AppGenerator() + + metadata = { + "name": "Test Model", + "version": "1.0", + "task": "segmentation", + "modality": "CT", + } + + context = generator._prepare_context( + model_id="MONAI/test_model", + metadata=metadata, + inference_config={}, + model_file=Path("models/model.ts"), + app_name=None, + ) + + assert context["model_id"] == "MONAI/test_model" + assert context["app_name"] == "TestModelApp" + assert context["task"] == "segmentation" + assert context["modality"] == "CT" + assert context["use_dicom"] is True + assert context["model_file"] == "models/model.ts" + + @patch.object(BundleDownloader, "download_bundle") + @patch.object(BundleDownloader, "get_bundle_metadata") + @patch.object(BundleDownloader, "get_inference_config") + @patch.object(BundleDownloader, "detect_model_file") + def test_generate_app(self, mock_detect_model, mock_get_inference, mock_get_metadata, mock_download): + """Test full app generation.""" + generator = AppGenerator() + + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + output_dir = temp_path / "output" + + # Mock bundle download + bundle_path = temp_path / "bundle" + bundle_path.mkdir() + mock_download.return_value = bundle_path + + # Mock metadata + mock_get_metadata.return_value = { + "name": "Test Model", + "version": "1.0", + "task": "segmentation", + "modality": "CT", + } + + # Mock inference config + mock_get_inference.return_value = {} + + # Mock model file + model_file = bundle_path / "models" / "model.ts" + model_file.parent.mkdir(parents=True) + model_file.touch() + mock_detect_model.return_value = model_file + + # Generate app + result = generator.generate_app("MONAI/test_model", output_dir) + + # Check generated files + assert result == output_dir + assert (output_dir / "app.py").exists() + assert (output_dir / "app.yaml").exists() + assert (output_dir / "requirements.txt").exists() + + def test_missing_metadata_uses_default(self): + """Test that missing metadata triggers default metadata creation.""" + generator = AppGenerator() + + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + output_dir = temp_path / "output" + + # Create a minimal bundle structure + bundle_path = temp_path / "model" + bundle_path.mkdir() + + # Mock the downloader to return bundle without metadata + with patch.object(generator.downloader, "download_bundle") as mock_download: + mock_download.return_value = bundle_path + + with patch.object(generator.downloader, "get_bundle_metadata") as mock_meta: + with patch.object(generator.downloader, "get_inference_config") as mock_inf: + with patch.object(generator.downloader, "detect_model_file") as mock_detect: + mock_meta.return_value = None # No metadata + mock_inf.return_value = {} + mock_detect.return_value = None + + with patch.object(generator, "_prepare_context") as mock_prepare: + with patch.object(generator, "_generate_app_py") as mock_app_py: + with patch.object(generator, "_generate_app_yaml") as mock_yaml: + with patch.object(generator, "_copy_additional_files") as mock_copy: + # Return a valid context + mock_prepare.return_value = { + "model_id": "MONAI/test_model", + "app_name": "TestApp", + "task": "segmentation", + } + + # This should trigger lines 73-74 and 438-439 + with patch( + "pipeline_generator.generator.app_generator.logger" + ) as mock_logger: + generator.generate_app( + "MONAI/test_model", + output_dir, + data_format="auto", + ) + + # Verify warning was logged + mock_logger.warning.assert_any_call( + "No metadata.json found in bundle, using defaults" + ) + + def test_inference_config_with_output_postfix(self): + """Test inference config with output_postfix string value.""" + generator = AppGenerator() + + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + output_dir = temp_path / "output" + + bundle_path = temp_path / "model" + bundle_path.mkdir() + + # Create inference config with output_postfix + inference_config = {"output_postfix": "_prediction"} # String value, not @variable + + metadata = {"name": "Test Model"} + + with patch.object(generator.downloader, "download_bundle") as mock_download: + mock_download.return_value = bundle_path + + with patch.object(generator.downloader, "get_bundle_metadata") as mock_meta: + with patch.object(generator.downloader, "get_inference_config") as mock_inf: + with patch.object(generator.downloader, "detect_model_file") as mock_detect: + mock_meta.return_value = metadata + mock_inf.return_value = inference_config # This triggers lines 194-196 + mock_detect.return_value = None + + with patch.object(generator, "_generate_app_py") as mock_app_py: + with patch.object(generator, "_generate_app_yaml") as mock_yaml: + with patch.object(generator, "_copy_additional_files") as mock_copy: + result = generator.generate_app( + "MONAI/test_model", + output_dir, + data_format="auto", + ) + + # Verify the output_postfix was extracted + call_args = mock_app_py.call_args[0][1] + assert call_args["output_postfix"] == "_prediction" + + def test_model_config_with_channel_first_override(self): + """Test model config with channel_first override in configs list.""" + from pipeline_generator.config.settings import ModelConfig + + generator = AppGenerator() + + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + output_dir = temp_path / "output" + + bundle_path = temp_path / "model" + bundle_path.mkdir() + + # Create model config with configs list + model_config = ModelConfig( + model_id="MONAI/test_model", + input_type="nifti", + output_type="nifti", + configs=[ + {"channel_first": True, "other": "value"}, + {"channel_first": False}, # Last one wins + ], + ) + + # Mock settings.get_model_config using patch + with patch("pipeline_generator.generator.app_generator.Settings.get_model_config") as mock_get_config: + mock_get_config.return_value = model_config + + with patch.object(generator.downloader, "download_bundle") as mock_download: + mock_download.return_value = bundle_path + + with patch.object(generator.downloader, "get_bundle_metadata") as mock_meta: + with patch.object(generator.downloader, "get_inference_config") as mock_inf: + with patch.object(generator.downloader, "detect_model_file") as mock_detect: + mock_meta.return_value = {"name": "Test"} + mock_inf.return_value = {} + mock_detect.return_value = None + + with patch.object(generator, "_generate_app_py") as mock_app_py: + with patch.object(generator, "_generate_app_yaml") as mock_yaml: + with patch.object(generator, "_copy_additional_files") as mock_copy: + generator.generate_app( + "MONAI/test_model", + output_dir, + data_format="auto", + ) + + # Verify channel_first logic is computed correctly + call_args = mock_app_py.call_args[0][1] + assert call_args["channel_first"] is False + + def test_metadata_with_numpy_pytorch_versions(self): + """Test metadata with numpy_version and pytorch_version.""" + generator = AppGenerator() + + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + output_dir = temp_path / "output" + + bundle_path = temp_path / "model" + bundle_path.mkdir() + + # Create metadata with version info + metadata = { + "name": "Test Model", + "numpy_version": "1.21.0", + "pytorch_version": "2.0.0", + } + + with patch.object(generator.downloader, "download_bundle") as mock_download: + mock_download.return_value = bundle_path + + with patch.object(generator.downloader, "get_bundle_metadata") as mock_meta: + with patch.object(generator.downloader, "get_inference_config") as mock_inf: + with patch.object(generator.downloader, "detect_model_file") as mock_detect: + with patch.object(generator.downloader, "organize_bundle_structure") as mock_organize: + mock_meta.return_value = metadata # This triggers lines 216, 218 + mock_inf.return_value = {} + mock_detect.return_value = None + + with patch.object(generator, "_generate_app_py") as mock_app_py: + with patch.object(generator, "_generate_app_yaml") as mock_yaml: + with patch.object(generator, "_copy_additional_files") as mock_copy: + generator.generate_app( + "MONAI/test_model", + output_dir, + data_format="auto", + ) + + # Verify dependencies were added + call_args = mock_copy.call_args[0][1] + assert "numpy==1.21.0" in call_args["extra_dependencies"] + assert "torch==2.0.0" in call_args["extra_dependencies"] + + def test_config_based_dependency_overrides(self): + """Test config-based dependency overrides prevent metadata conflicts.""" + from pipeline_generator.config.settings import Endpoint, ModelConfig, Settings + + # Mock settings with config override for a model + model_config = ModelConfig( + model_id="MONAI/test_model", + input_type="nifti", + output_type="nifti", + dependencies=["torch>=1.11.0", "numpy>=1.21.0", "monai>=1.3.0"], + ) + + endpoint = Endpoint( + organization="MONAI", base_url="https://huggingface.co", description="Test", models=[model_config] + ) + + settings = Settings(endpoints=[endpoint]) + generator = AppGenerator(settings) + + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + output_dir = temp_path / "output" + bundle_path = temp_path / "model" + bundle_path.mkdir() + + # Mock metadata with conflicting versions + metadata = { + "name": "Test Model", + "numpy_version": "1.20.0", # Older version + "pytorch_version": "1.10.0", # Incompatible version + "monai_version": "0.8.0", # Old MONAI version + } + + with patch.object(generator.downloader, "download_bundle") as mock_download: + mock_download.return_value = bundle_path + + with patch.object(generator.downloader, "get_bundle_metadata") as mock_meta: + with patch.object(generator.downloader, "get_inference_config") as mock_inf: + with patch.object(generator.downloader, "detect_model_file") as mock_detect: + with patch.object(generator.downloader, "organize_bundle_structure") as mock_organize: + mock_meta.return_value = metadata + mock_inf.return_value = {} + mock_detect.return_value = None + + with patch.object(generator, "_generate_app_py") as mock_app_py: + with patch.object(generator, "_generate_app_yaml") as mock_yaml: + with patch.object(generator, "_copy_additional_files") as mock_copy: + generator.generate_app( + "MONAI/test_model", + output_dir, + data_format="auto", + ) + + call_args = mock_copy.call_args[0][1] + + # Config dependencies should be used instead of metadata + assert "torch>=1.11.0" in call_args["extra_dependencies"] + assert "numpy>=1.21.0" in call_args["extra_dependencies"] + assert "monai>=1.3.0" in call_args["extra_dependencies"] + + # Old metadata versions should NOT be included + assert "torch==1.10.0" not in call_args["extra_dependencies"] + assert "numpy==1.20.0" not in call_args["extra_dependencies"] + + # MONAI version should be removed from metadata to prevent template conflict + assert "monai_version" not in call_args["metadata"] + + # Verify bundle structure was organized + mock_organize.assert_called_once() + + def test_dependency_conflict_resolution_no_config(self): + """Test that without config overrides, metadata versions are used.""" + generator = AppGenerator() # No settings, no config overrides + + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + output_dir = temp_path / "output" + bundle_path = temp_path / "model" + bundle_path.mkdir() + + metadata = { + "name": "Test Model", + "numpy_version": "1.21.0", + "pytorch_version": "1.12.0", + "monai_version": "1.0.0", + } + + with patch.object(generator.downloader, "download_bundle") as mock_download: + mock_download.return_value = bundle_path + + with patch.object(generator.downloader, "get_bundle_metadata") as mock_meta: + with patch.object(generator.downloader, "get_inference_config") as mock_inf: + with patch.object(generator.downloader, "detect_model_file") as mock_detect: + with patch.object(generator.downloader, "organize_bundle_structure") as mock_organize: + mock_meta.return_value = metadata + mock_inf.return_value = {} + mock_detect.return_value = None + + with patch.object(generator, "_generate_app_py") as mock_app_py: + with patch.object(generator, "_generate_app_yaml") as mock_yaml: + with patch.object(generator, "_copy_additional_files") as mock_copy: + generator.generate_app( + "MONAI/test_model", + output_dir, + data_format="auto", + ) + + call_args = mock_copy.call_args[0][1] + + # Should use metadata versions when no config + assert "numpy==1.21.0" in call_args["extra_dependencies"] + assert "torch==1.12.0" in call_args["extra_dependencies"] + + # MONAI version should be moved from metadata to extra_dependencies + assert "monai==1.0.0" in call_args["extra_dependencies"] + assert "monai_version" not in call_args["metadata"] + + def test_monai_version_handling_in_app_generator(self): + """Test that MONAI version logic is correctly handled in app generator (moved from template).""" + generator = AppGenerator() + + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + output_dir = temp_path / "output" + bundle_path = temp_path / "model" + bundle_path.mkdir() + + # Test case 1: Config has MONAI - should not add metadata version + with patch.object(generator.downloader, "download_bundle") as mock_download: + mock_download.return_value = bundle_path + + with patch.object(generator.downloader, "get_bundle_metadata") as mock_meta: + with patch.object(generator.downloader, "get_inference_config") as mock_inf: + with patch.object(generator.downloader, "detect_model_file") as mock_detect: + with patch.object(generator.downloader, "organize_bundle_structure") as mock_organize: + # Mock model config with MONAI dependency + from pipeline_generator.config.settings import Endpoint, ModelConfig, Settings + + model_config = ModelConfig( + model_id="MONAI/test_model", + input_type="nifti", + output_type="nifti", + dependencies=["monai>=1.3.0"], + ) + endpoint = Endpoint( + organization="MONAI", + base_url="https://huggingface.co", + description="Test", + models=[model_config], + ) + settings = Settings(endpoints=[endpoint]) + generator_with_config = AppGenerator(settings) + + mock_meta.return_value = {"monai_version": "0.8.0"} + mock_inf.return_value = {} + mock_detect.return_value = None + + context = generator_with_config._prepare_context( + "MONAI/test_model", + {"monai_version": "0.8.0"}, + {}, + None, + None, + "auto", + "segmentation", + None, + None, + model_config, # Pass the model config + ) + + # Should have config MONAI but not metadata MONAI + assert "monai>=1.3.0" in context["extra_dependencies"] + assert "monai==0.8.0" not in context["extra_dependencies"] + assert "monai_version" not in context["metadata"] + + # Test case 2: No config MONAI - should add metadata version + generator_no_config = AppGenerator() # No settings + context2 = generator_no_config._prepare_context( + "MONAI/test_model", + {"monai_version": "1.0.0"}, + {}, + None, + None, + "auto", + "segmentation", + None, + None, + None, # No model config + ) + + # Should add metadata MONAI version to extra_dependencies + assert "monai==1.0.0" in context2["extra_dependencies"] + assert "monai_version" not in context2["metadata"] + + # Test case 3: No config and no metadata - should add fallback + context3 = generator_no_config._prepare_context( + "MONAI/test_model", {}, {}, None, None, "auto", "segmentation", None, None, None # No model config + ) + + # Should add fallback MONAI version + assert "monai>=1.5.0" in context3["extra_dependencies"] + + def test_inference_config_with_loadimage_transform(self): + """Test _detect_data_format with LoadImaged transform.""" + generator = AppGenerator() + + # Create inference config with LoadImaged transform + inference_config = { + "preprocessing": { + "transforms": [ + {"_target_": "monai.transforms.LoadImaged", "keys": ["image"]}, + {"_target_": "monai.transforms.EnsureChannelFirstd"}, + ] + } + } + + # This should return False (NIfTI format) - covers lines 259-264 + result = generator._detect_data_format(inference_config, "CT") + assert result is False + + def test_inference_config_with_string_transforms(self): + """Test _detect_data_format with string transforms expression.""" + generator = AppGenerator() + + # Create inference config with string transforms (like spleen_deepedit_annotation) + inference_config = { + "preprocessing": { + "_target_": "Compose", + "transforms": "$@preprocessing_transforms + @deepedit_transforms + @extra_transforms", + }, + "preprocessing_transforms": [ + {"_target_": "LoadImaged", "keys": "image"}, + {"_target_": "EnsureChannelFirstd", "keys": "image"}, + ], + } + + # This should return False (NIfTI format) because LoadImaged is found in config string + result = generator._detect_data_format(inference_config, "CT") + assert result is False + + def test_inference_config_with_string_transforms_no_loadimage(self): + """Test _detect_data_format with string transforms expression without LoadImaged.""" + generator = AppGenerator() + + # Create inference config with string transforms but no LoadImaged + inference_config = { + "preprocessing": {"_target_": "Compose", "transforms": "$@preprocessing_transforms + @other_transforms"}, + "preprocessing_transforms": [ + {"_target_": "SomeOtherTransform", "keys": "image"}, + {"_target_": "EnsureChannelFirstd", "keys": "image"}, + ], + } + + # This should return True (DICOM format) for CT modality when no LoadImaged found + result = generator._detect_data_format(inference_config, "CT") + assert result is True + + def test_detect_model_type_pathology(self): + """Test _detect_model_type for pathology models.""" + generator = AppGenerator() + + # Test pathology detection by model ID - covers line 319 + assert generator._detect_model_type("LGAI-EXAONE/EXAONEPath", {}) == "pathology" + assert generator._detect_model_type("MONAI/pathology_model", {}) == "pathology" + + # Test pathology detection by metadata - covers line 333 + metadata = {"task": "pathology classification"} + assert generator._detect_model_type("MONAI/some_model", metadata) == "pathology" + + def test_detect_model_type_multimodal_llm(self): + """Test _detect_model_type for multimodal LLM models.""" + generator = AppGenerator() + + # Test LLM detection - covers line 323 + assert generator._detect_model_type("MONAI/Llama3-VILA-M3-3B", {}) == "multimodal_llm" + assert generator._detect_model_type("MONAI/vila_model", {}) == "multimodal_llm" + + def test_detect_model_type_multimodal(self): + """Test _detect_model_type for multimodal models.""" + generator = AppGenerator() + + # Test multimodal detection by model ID - covers line 327 + assert generator._detect_model_type("MONAI/chat_model", {}) == "multimodal" + assert generator._detect_model_type("MONAI/multimodal_seg", {}) == "multimodal" + + # Test multimodal detection by metadata - covers line 335 + metadata = {"task": "medical chat"} + assert generator._detect_model_type("MONAI/some_model", metadata) == "multimodal" + + metadata = {"task": "visual qa"} + assert generator._detect_model_type("MONAI/some_model", metadata) == "multimodal" + + def test_model_config_with_dict_configs(self): + """Test model config with configs as dict instead of list.""" + from pipeline_generator.config.settings import ModelConfig + + generator = AppGenerator() + + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + output_dir = temp_path / "output" + + bundle_path = temp_path / "model" + bundle_path.mkdir() + + # Create model config with configs dict - covers line 210 + model_config = ModelConfig( + model_id="MONAI/test_model", + input_type="nifti", + output_type="nifti", + configs={"channel_first": True}, # Dict instead of list + ) + + # Mock settings.get_model_config using patch + with patch("pipeline_generator.generator.app_generator.Settings.get_model_config") as mock_get_config: + mock_get_config.return_value = model_config + + with patch.object(generator.downloader, "download_bundle") as mock_download: + mock_download.return_value = bundle_path + + with patch.object(generator.downloader, "get_bundle_metadata") as mock_meta: + with patch.object(generator.downloader, "get_inference_config") as mock_inf: + with patch.object(generator.downloader, "detect_model_file") as mock_detect: + mock_meta.return_value = {"name": "Test"} + mock_inf.return_value = {} + mock_detect.return_value = None + + with patch.object(generator, "_generate_app_py") as mock_app_py: + with patch.object(generator, "_generate_app_yaml") as mock_yaml: + with patch.object(generator, "_copy_additional_files") as mock_copy: + generator.generate_app( + "MONAI/test_model", + output_dir, + data_format="auto", + ) + + call_args = mock_app_py.call_args[0][1] + assert call_args["channel_first"] is True + + def test_channel_first_logic_refactoring(self): + """Test the refactored channel_first logic works correctly.""" + generator = AppGenerator() + + # Test case 1: image input, non-classification task -> should be False + context1 = generator._prepare_context( + model_id="test/model", + metadata={"task": "segmentation", "name": "Test Model"}, + inference_config={}, + model_file=None, + app_name="TestApp", + input_type="image", + output_type="nifti", + ) + assert context1["channel_first"] is False + + # Test case 2: image input, classification task -> should be True + context2 = generator._prepare_context( + model_id="test/model", + metadata={"task": "classification", "name": "Test Model"}, + inference_config={}, + model_file=None, + app_name="TestApp", + input_type="image", + output_type="json", + ) + assert context2["channel_first"] is True + + # Test case 3: dicom input -> should be True + context3 = generator._prepare_context( + model_id="test/model", + metadata={"task": "segmentation", "name": "Test Model"}, + inference_config={}, + model_file=None, + app_name="TestApp", + input_type="dicom", + output_type="nifti", + ) + assert context3["channel_first"] is True + + # Test case 4: nifti input -> should be True + context4 = generator._prepare_context( + model_id="test/model", + metadata={"task": "segmentation", "name": "Test Model"}, + inference_config={}, + model_file=None, + app_name="TestApp", + input_type="nifti", + output_type="nifti", + ) + assert context4["channel_first"] is True + + def test_get_default_metadata(self): + """Test _get_default_metadata method directly.""" + generator = AppGenerator() + + # Test default metadata generation - covers lines 438-439 + metadata = generator._get_default_metadata("MONAI/spleen_ct_segmentation") + + assert metadata["name"] == "Spleen Ct Segmentation" + assert metadata["version"] == "1.0" + assert metadata["task"] == "segmentation" + assert metadata["modality"] == "CT" + assert "spleen_ct_segmentation" in metadata["description"] + + @patch.object(BundleDownloader, "download_bundle") + @patch.object(BundleDownloader, "get_bundle_metadata") + @patch.object(BundleDownloader, "get_inference_config") + @patch.object(BundleDownloader, "detect_model_file") + def test_nifti_segmentation_imports(self, mock_detect_model, mock_get_inference, mock_get_metadata, mock_download): + """Test that NIfTI segmentation apps have required imports.""" + generator = AppGenerator() + + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + output_dir = temp_path / "output" + + # Mock bundle download + bundle_path = temp_path / "bundle" + bundle_path.mkdir() + mock_download.return_value = bundle_path + + # Mock metadata for NIfTI segmentation + mock_get_metadata.return_value = { + "name": "Spleen CT Segmentation", + "version": "1.0", + "task": "segmentation", + "modality": "CT", + } + + # Mock inference config (minimal) + mock_get_inference.return_value = {} + + # Mock model file (TorchScript) + model_file = bundle_path / "models" / "model.ts" + model_file.parent.mkdir(parents=True) + model_file.touch() + mock_detect_model.return_value = model_file + + # Generate app + generator.generate_app("MONAI/spleen_ct_segmentation", output_dir) + + # Read generated app.py + app_file = output_dir / "app.py" + assert app_file.exists() + app_content = app_file.read_text() + + # Check critical imports for MonaiBundleInferenceOperator + assert ( + "from monai.deploy.core.domain import Image" in app_content + ), "Image import missing - required for MonaiBundleInferenceOperator" + assert ( + "from monai.deploy.core.io_type import IOType" in app_content + ), "IOType import missing - required for MonaiBundleInferenceOperator" + assert "IOMapping" in app_content, "IOMapping import missing - required for MonaiBundleInferenceOperator" + + # Check operator imports + assert "from generic_directory_scanner_operator import GenericDirectoryScanner" in app_content + assert "from monai.deploy.operators.nii_data_loader_operator import NiftiDataLoader" in app_content + assert "from nifti_writer_operator import NiftiWriter" in app_content + assert "from monai.deploy.operators.monai_bundle_inference_operator import" in app_content + + # Check that the required operator files are physically copied (Phase 7 verification) + assert ( + output_dir / "generic_directory_scanner_operator.py" + ).exists(), "GenericDirectoryScanner operator file not copied" + assert (output_dir / "nifti_writer_operator.py").exists(), "NiftiWriter operator file not copied" + + @patch.object(BundleDownloader, "download_bundle") + @patch.object(BundleDownloader, "get_bundle_metadata") + @patch.object(BundleDownloader, "get_inference_config") + @patch.object(BundleDownloader, "detect_model_file") + def test_image_classification_imports( + self, mock_detect_model, mock_get_inference, mock_get_metadata, mock_download + ): + """Test that image classification apps have required imports.""" + generator = AppGenerator() + + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + output_dir = temp_path / "output" + + # Mock bundle download + bundle_path = temp_path / "bundle" + bundle_path.mkdir() + mock_download.return_value = bundle_path + + # Mock metadata for classification + mock_get_metadata.return_value = { + "name": "Breast Density Classification", + "version": "1.0", + "task": "Mammographic Breast Density Classification (BI-RADS)", + "modality": "MG", + "data_type": "jpeg", + } + + # Mock inference config + mock_get_inference.return_value = {} + + # Mock model file (PyTorch) + model_file = bundle_path / "models" / "model.pt" + model_file.parent.mkdir(parents=True) + model_file.touch() + mock_detect_model.return_value = model_file + + # Generate app with detected image/json format + generator.generate_app("MONAI/breast_density_classification", output_dir) + + # Read generated app.py + app_file = output_dir / "app.py" + assert app_file.exists() + app_content = app_file.read_text() + + # Check critical imports + assert "from monai.deploy.core.domain import Image" in app_content, "Image import missing" + assert "from monai.deploy.core.io_type import IOType" in app_content, "IOType import missing" + + # Check operator imports + assert "from generic_directory_scanner_operator import GenericDirectoryScanner" in app_content + assert "from image_file_loader_operator import ImageFileLoader" in app_content + assert "from json_results_writer_operator import JSONResultsWriter" in app_content + assert "from monai_classification_operator import MonaiClassificationOperator" in app_content + + # Check that the required operator files are physically copied (Phase 7 verification) + assert ( + output_dir / "generic_directory_scanner_operator.py" + ).exists(), "GenericDirectoryScanner operator file not copied" + assert (output_dir / "image_file_loader_operator.py").exists(), "ImageFileLoader operator file not copied" + assert ( + output_dir / "json_results_writer_operator.py" + ).exists(), "JSONResultsWriter operator file not copied" + assert ( + output_dir / "monai_classification_operator.py" + ).exists(), "MonaiClassificationOperator operator file not copied" + + @patch.object(BundleDownloader, "download_bundle") + @patch.object(BundleDownloader, "get_bundle_metadata") + @patch.object(BundleDownloader, "get_inference_config") + @patch.object(BundleDownloader, "detect_model_file") + def test_vlm_model_imports_and_operators( + self, mock_detect_model, mock_get_inference, mock_get_metadata, mock_download + ): + """Test that VLM apps have required imports and operators copied (Phase 7 verification).""" + generator = AppGenerator() + + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + output_dir = temp_path / "output" + + # Mock bundle download + bundle_path = temp_path / "bundle" + bundle_path.mkdir() + mock_download.return_value = bundle_path + + # Mock metadata for VLM model + mock_get_metadata.return_value = { + "name": "Llama3-VILA-M3-3B", + "version": "1.0", + "task": "vlm", + "modality": "multimodal", + } + + # Mock inference config (VLM doesn't have traditional inference config) + mock_get_inference.return_value = {} + + # Mock: No traditional model file for VLM + mock_detect_model.return_value = None + + # Generate app + generator.generate_app("MONAI/Llama3-VILA-M3-3B", output_dir) + + # Read generated app.py + app_file = output_dir / "app.py" + assert app_file.exists() + app_content = app_file.read_text() + + # Check VLM-specific imports + assert "from prompts_loader_operator import PromptsLoaderOperator" in app_content + assert "from llama3_vila_inference_operator import Llama3VILAInferenceOperator" in app_content + assert "from vlm_results_writer_operator import VLMResultsWriterOperator" in app_content + + # Check that the VLM operator files are physically copied (Phase 7 verification) + assert ( + output_dir / "prompts_loader_operator.py" + ).exists(), "PromptsLoaderOperator operator file not copied" + assert ( + output_dir / "llama3_vila_inference_operator.py" + ).exists(), "Llama3VILAInferenceOperator operator file not copied" + assert ( + output_dir / "vlm_results_writer_operator.py" + ).exists(), "VLMResultsWriterOperator operator file not copied" + + # Verify that non-VLM operators are NOT copied for VLM models + assert not ( + output_dir / "nifti_writer_operator.py" + ).exists(), "NiftiWriter should not be copied for VLM models" + assert not ( + output_dir / "monai_classification_operator.py" + ).exists(), "MonaiClassificationOperator should not be copied for VLM models" + + @patch.object(BundleDownloader, "download_bundle") + @patch.object(BundleDownloader, "get_bundle_metadata") + @patch.object(BundleDownloader, "get_inference_config") + @patch.object(BundleDownloader, "detect_model_file") + def test_dicom_segmentation_imports(self, mock_detect_model, mock_get_inference, mock_get_metadata, mock_download): + """Test that DICOM segmentation apps have required imports.""" + generator = AppGenerator() + + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + output_dir = temp_path / "output" + + # Mock bundle download + bundle_path = temp_path / "bundle" + bundle_path.mkdir() + mock_download.return_value = bundle_path + + # Mock metadata for DICOM segmentation + mock_get_metadata.return_value = { + "name": "Spleen CT Segmentation", + "version": "1.0", + "task": "Automated Spleen Segmentation in CT Images", + "modality": "CT", + } + + # Mock inference config + mock_get_inference.return_value = {} + + # Mock model file + model_file = bundle_path / "models" / "model.ts" + model_file.parent.mkdir(parents=True) + model_file.touch() + mock_detect_model.return_value = model_file + + # Generate app with DICOM format + generator.generate_app("MONAI/spleen_ct_segmentation", output_dir, data_format="dicom") + + # Read generated app.py + app_file = output_dir / "app.py" + assert app_file.exists() + app_content = app_file.read_text() + + # Check critical imports + assert ( + "from monai.deploy.core.domain import Image" in app_content + ), "Image import missing - required for MonaiBundleInferenceOperator" + assert ( + "from monai.deploy.core.io_type import IOType" in app_content + ), "IOType import missing - required for MonaiBundleInferenceOperator" + + # Check DICOM-specific imports + assert "from pydicom.sr.codedict import codes" in app_content + assert "from monai.deploy.conditions import CountCondition" in app_content + assert ( + "from monai.deploy.operators.dicom_data_loader_operator import DICOMDataLoaderOperator" in app_content + ) + assert ( + "from monai.deploy.operators.dicom_seg_writer_operator import DICOMSegmentationWriterOperator" + in app_content + ) + assert "from monai.deploy.operators.stl_conversion_operator import STLConversionOperator" in app_content + + def test_imports_syntax_validation(self): + """Test that generated apps have valid Python syntax.""" + generator = AppGenerator() + + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + output_dir = temp_path / "output" + + # Create a minimal test by mocking all dependencies + with ( + patch.object(BundleDownloader, "download_bundle") as mock_download, + patch.object(BundleDownloader, "get_bundle_metadata") as mock_metadata, + patch.object(BundleDownloader, "get_inference_config") as mock_config, + patch.object(BundleDownloader, "detect_model_file") as mock_detect, + ): + bundle_path = temp_path / "bundle" + bundle_path.mkdir() + mock_download.return_value = bundle_path + mock_metadata.return_value = {"name": "Test", "task": "segmentation"} + mock_config.return_value = {} + model_file = bundle_path / "models" / "model.ts" + model_file.parent.mkdir(parents=True) + model_file.touch() + mock_detect.return_value = model_file + + generator.generate_app("MONAI/test", output_dir) + + # Try to compile the generated Python file + app_file = output_dir / "app.py" + app_content = app_file.read_text() + + try: + compile(app_content, str(app_file), "exec") + except SyntaxError as e: + pytest.fail(f"Generated app.py has syntax error: {e}") + + def test_monai_bundle_inference_operator_requirements(self): + """Test that apps using MonaiBundleInferenceOperator have all required imports.""" + generator = AppGenerator() + + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + output_dir = temp_path / "output" + + # Test different scenarios that use MonaiBundleInferenceOperator + test_cases = [ + # NIfTI segmentation (original failing case) + { + "metadata": { + "name": "Test Segmentation", + "task": "segmentation", + "modality": "CT", + }, + "model_file": "model.ts", + "format": "auto", + }, + # NIfTI with different task description + { + "metadata": { + "name": "Organ Detection", + "task": "detection", + "modality": "MR", + }, + "model_file": "model.ts", + "format": "nifti", + }, + ] + + for test_case in test_cases: + with ( + patch.object(BundleDownloader, "download_bundle") as mock_download, + patch.object(BundleDownloader, "get_bundle_metadata") as mock_metadata, + patch.object(BundleDownloader, "get_inference_config") as mock_config, + patch.object(BundleDownloader, "detect_model_file") as mock_detect, + ): + bundle_path = temp_path / f"bundle_{test_case['format']}" + bundle_path.mkdir() + mock_download.return_value = bundle_path + mock_metadata.return_value = test_case["metadata"] + mock_config.return_value = {} + + model_file = bundle_path / "models" / test_case["model_file"] + model_file.parent.mkdir(parents=True) + model_file.touch() + mock_detect.return_value = model_file + + output_subdir = output_dir / f"test_{test_case['format']}" + generator.generate_app("MONAI/test", output_subdir, data_format=test_case["format"]) + + # Read and check generated app + app_file = output_subdir / "app.py" + app_content = app_file.read_text() + + # If MonaiBundleInferenceOperator is used, these imports must be present + if "MonaiBundleInferenceOperator" in app_content: + assert ( + "from monai.deploy.core.domain import Image" in app_content + ), f"Image import missing for {test_case['format']} format" + assert ( + "from monai.deploy.core.io_type import IOType" in app_content + ), f"IOType import missing for {test_case['format']} format" + assert ( + "IOMapping" in app_content + ), "IOMapping must be imported when using MonaiBundleInferenceOperator" diff --git a/tools/pipeline-generator/tests/test_hub_client.py b/tools/pipeline-generator/tests/test_hub_client.py new file mode 100644 index 00000000..d01e7a4b --- /dev/null +++ b/tools/pipeline-generator/tests/test_hub_client.py @@ -0,0 +1,366 @@ +# Copyright 2025 MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for HuggingFace Hub client.""" + +from datetime import datetime +from unittest.mock import Mock, patch + +from huggingface_hub.utils import HfHubHTTPError +from pipeline_generator.core.hub_client import HuggingFaceClient + + +class SimpleModelData: + """Simple class to simulate HuggingFace model data.""" + + def __init__(self, **kwargs): + for key, value in kwargs.items(): + setattr(self, key, value) + + +class TestHuggingFaceClient: + """Test HuggingFace client functionality.""" + + def setup_method(self): + """Set up test fixtures.""" + self.client = HuggingFaceClient() + + @patch("pipeline_generator.core.hub_client.list_models") + def test_list_models_from_organization_success(self, mock_list_models): + """Test successfully listing models from organization.""" + # Mock model data + mock_model1 = SimpleModelData( + modelId="MONAI/spleen_ct_segmentation", + author="MONAI", + downloads=100, + likes=10, + created_at=datetime(2023, 1, 1), + lastModified=datetime(2023, 12, 1), + tags=["medical", "segmentation"], + siblings=[Mock(rfilename="configs/metadata.json")], + ) + + mock_model2 = SimpleModelData( + modelId="MONAI/liver_segmentation", + author="MONAI", + downloads=50, + likes=5, + created_at=datetime(2023, 2, 1), + lastModified=datetime(2023, 11, 1), + tags=["medical"], + siblings=[], + ) + + mock_list_models.return_value = [mock_model1, mock_model2] + + # Call the method + models = self.client.list_models_from_organization("MONAI") + + # Verify results + assert len(models) == 2 + assert models[0].model_id == "MONAI/spleen_ct_segmentation" + assert models[0].is_monai_bundle is True # Has metadata.json + assert models[1].model_id == "MONAI/liver_segmentation" + assert models[1].is_monai_bundle is False # No metadata.json + + @patch("pipeline_generator.core.hub_client.list_models") + def test_list_models_from_organization_empty(self, mock_list_models): + """Test listing models from organization with no results.""" + mock_list_models.return_value = [] + + models = self.client.list_models_from_organization("NonExistent") + + assert len(models) == 0 + + @patch("pipeline_generator.core.hub_client.list_models") + def test_list_models_from_organization_error(self, mock_list_models): + """Test handling errors when listing models.""" + mock_list_models.side_effect = Exception("API Error") + + models = self.client.list_models_from_organization("MONAI") + + assert len(models) == 0 # Should return empty list on error + + @patch("pipeline_generator.core.hub_client.model_info") + def test_get_model_info_success(self, mock_model_info): + """Test successfully getting model info.""" + # Mock model data + mock_model = SimpleModelData( + modelId="MONAI/spleen_ct_segmentation", + author="MONAI", + downloads=100, + likes=10, + created_at=datetime(2023, 1, 1), + lastModified=datetime(2023, 12, 1), + tags=["medical", "segmentation"], + siblings=[Mock(rfilename="configs/metadata.json")], + cardData={"description": "Spleen segmentation model"}, + ) + + mock_model_info.return_value = mock_model + + # Call the method + model = self.client.get_model_info("MONAI/spleen_ct_segmentation") + + # Verify results + assert model is not None + assert model.model_id == "MONAI/spleen_ct_segmentation" + assert model.author == "MONAI" + assert model.is_monai_bundle is True + assert model.description == "Spleen segmentation model" + + @patch("pipeline_generator.core.hub_client.model_info") + def test_get_model_info_not_found(self, mock_model_info): + """Test getting model info for non-existent model.""" + mock_model_info.side_effect = HfHubHTTPError("Model not found", response=Mock(status_code=404)) + + model = self.client.get_model_info("MONAI/nonexistent") + + assert model is None + + @patch("pipeline_generator.core.hub_client.model_info") + def test_get_model_info_error(self, mock_model_info): + """Test handling errors when getting model info.""" + mock_model_info.side_effect = Exception("API Error") + + model = self.client.get_model_info("MONAI/spleen_ct_segmentation") + + assert model is None + + def test_extract_model_info_with_name(self): + """Test parsing model info with explicit name.""" + mock_model = SimpleModelData( + modelId="MONAI/test_model", + name="Test Model", + author="MONAI", + downloads=100, + likes=10, + created_at=datetime(2023, 1, 1), + lastModified=datetime(2023, 12, 1), + tags=["test"], + siblings=[], + ) + + model = self.client._extract_model_info(mock_model) + + assert model.model_id == "MONAI/test_model" + assert model.name == "Test Model" + assert model.display_name == "Test Model" + + def test_extract_model_info_without_name(self): + """Test parsing model info without explicit name.""" + mock_model = SimpleModelData( + modelId="MONAI/test_model", + author=None, + downloads=None, + likes=None, + created_at=None, + lastModified=None, + tags=[], + siblings=[], + ) + + model = self.client._extract_model_info(mock_model) + + assert model.model_id == "MONAI/test_model" + assert model.name == "MONAI/test_model" # Uses modelId as fallback + assert model.author is None + + def test_extract_model_info_bundle_detection(self): + """Test MONAI bundle detection during parsing.""" + # Test with metadata.json in siblings + mock_model = SimpleModelData( + modelId="MONAI/test_bundle", + author="MONAI", + downloads=100, + likes=10, + created_at=datetime(2023, 1, 1), + lastModified=datetime(2023, 12, 1), + tags=[], + siblings=[ + Mock(rfilename="configs/metadata.json"), + Mock(rfilename="models/model.pt"), + ], + ) + model = self.client._extract_model_info(mock_model) + assert model.is_monai_bundle is True + + # Test without metadata.json + mock_model.siblings = [Mock(rfilename="models/model.pt")] + model = self.client._extract_model_info(mock_model) + assert model.is_monai_bundle is False + + def test_extract_model_info_missing_siblings(self): + """Test parsing model info when siblings attribute is missing.""" + mock_model = SimpleModelData( + modelId="MONAI/test_model", + author="MONAI", + downloads=100, + likes=10, + created_at=datetime(2023, 1, 1), + lastModified=datetime(2023, 12, 1), + tags=[], + ) + # Don't set siblings attribute + + model = self.client._extract_model_info(mock_model) + + assert model.is_monai_bundle is False # Should default to False on error + + def test_extract_model_info_with_description(self): + """Test parsing model info with description in cardData.""" + mock_model = SimpleModelData( + modelId="MONAI/test_model", + author="MONAI", + downloads=100, + likes=10, + created_at=datetime(2023, 1, 1), + lastModified=datetime(2023, 12, 1), + tags=["medical"], + siblings=[], + cardData={"description": "This is a test model"}, + ) + + model = self.client._extract_model_info(mock_model) + + assert model.description == "This is a test model" + + def test_extract_model_info_missing_optional_attributes(self): + """Test parsing model info with missing optional attributes.""" + mock_model = SimpleModelData(modelId="MONAI/test_model", siblings=[]) + + model = self.client._extract_model_info(mock_model) + + assert model.model_id == "MONAI/test_model" + assert model.author is None + assert model.downloads is None + assert model.likes is None + assert model.created_at is None + assert model.updated_at is None + assert model.tags == [] + + def test_list_models_from_endpoints_with_organization(self): + """Test listing models from endpoints with organization.""" + from pipeline_generator.config.settings import Endpoint + + # Create test endpoints + endpoints = [ + Endpoint( + organization="MONAI", + base_url="https://huggingface.co", + description="Test org", + models=[], + ) + ] + + # Mock the list_models_from_organization method + with patch.object(self.client, "list_models_from_organization") as mock_list: + mock_list.return_value = [Mock(model_id="MONAI/test_model")] + + result = self.client.list_models_from_endpoints(endpoints) + + assert len(result) == 1 + mock_list.assert_called_once_with("MONAI") + + def test_list_models_from_endpoints_with_model_id(self): + """Test listing models from endpoints with specific model_id.""" + from pipeline_generator.config.settings import Endpoint + + # Create test endpoints with model_id + endpoints = [ + Endpoint( + model_id="MONAI/specific_model", + base_url="https://huggingface.co", + description="Test model", + models=[], + ) + ] + + # Mock the get_model_info method + with patch.object(self.client, "get_model_info") as mock_get: + mock_model = Mock(model_id="MONAI/specific_model") + mock_get.return_value = mock_model + + result = self.client.list_models_from_endpoints(endpoints) + + assert len(result) == 1 + assert result[0] == mock_model + mock_get.assert_called_once_with("MONAI/specific_model") + + def test_list_models_from_endpoints_model_not_found(self): + """Test listing models when specific model is not found.""" + from pipeline_generator.config.settings import Endpoint + + endpoints = [ + Endpoint( + model_id="MONAI/missing_model", + base_url="https://huggingface.co", + description="Missing model", + models=[], + ) + ] + + # Mock get_model_info to return None + with patch.object(self.client, "get_model_info") as mock_get: + mock_get.return_value = None + + result = self.client.list_models_from_endpoints(endpoints) + + assert len(result) == 0 + mock_get.assert_called_once_with("MONAI/missing_model") + + def test_extract_model_info_siblings_exception(self): + """Test _extract_model_info handles exception in siblings check.""" + + # Create a mock model that will raise exception when accessing siblings + class MockModelWithException: + def __init__(self): + self.modelId = "test/model" + self.tags = [] + self.downloads = 100 + self.likes = 10 + self.name = "Test Model" + self.author = "test" + self.description = None + self.created_at = None + self.lastModified = None + + @property + def siblings(self): + raise Exception("Test error") + + mock_model = MockModelWithException() + + # Should not raise, just catch and continue + result = self.client._extract_model_info(mock_model) + + assert result.is_monai_bundle is False + + def test_extract_model_info_with_card_data_preference(self): + """Test _extract_model_info prefers description from cardData.""" + mock_model = SimpleModelData( + modelId="test/model", + tags=[], + downloads=100, + likes=10, + name="Test Model", + author="test", + description="Direct description", + cardData={"description": "Card description"}, + created_at=None, + lastModified=None, + siblings=[], + ) + + result = self.client._extract_model_info(mock_model) + + # Should prefer cardData description + assert result.description == "Card description" diff --git a/tools/pipeline-generator/tests/test_models.py b/tools/pipeline-generator/tests/test_models.py new file mode 100644 index 00000000..344b2917 --- /dev/null +++ b/tools/pipeline-generator/tests/test_models.py @@ -0,0 +1,74 @@ +# Copyright 2025 MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for ModelInfo data model.""" + +from datetime import datetime + +from pipeline_generator.core.models import ModelInfo + + +class TestModelInfo: + """Test ModelInfo data model.""" + + def test_basic_model_creation(self): + """Test creating a basic ModelInfo object.""" + model = ModelInfo(model_id="MONAI/spleen_ct_segmentation", name="Spleen CT Segmentation") + + assert model.model_id == "MONAI/spleen_ct_segmentation" + assert model.name == "Spleen CT Segmentation" + assert model.is_monai_bundle is False + assert model.tags == [] + + def test_display_name_with_name(self): + """Test display_name property when name is provided.""" + model = ModelInfo(model_id="MONAI/test_model", name="Test Model") + + assert model.display_name == "Test Model" + + def test_display_name_without_name(self): + """Test display_name property when name is not provided.""" + model = ModelInfo(model_id="MONAI/spleen_ct_segmentation", name="") + + assert model.display_name == "Spleen Ct Segmentation" + + def test_short_id(self): + """Test short_id property.""" + model = ModelInfo(model_id="MONAI/spleen_ct_segmentation", name="Test") + + assert model.short_id == "spleen_ct_segmentation" + + def test_full_model_creation(self): + """Test creating a ModelInfo with all fields.""" + now = datetime.now() + model = ModelInfo( + model_id="MONAI/test_model", + name="Test Model", + author="MONAI", + description="A test model", + downloads=100, + likes=10, + created_at=now, + updated_at=now, + tags=["medical", "segmentation"], + is_monai_bundle=True, + bundle_metadata={"version": "1.0"}, + ) + + assert model.author == "MONAI" + assert model.description == "A test model" + assert model.downloads == 100 + assert model.likes == 10 + assert model.created_at == now + assert model.updated_at == now + assert model.tags == ["medical", "segmentation"] + assert model.is_monai_bundle is True + assert model.bundle_metadata == {"version": "1.0"} diff --git a/tools/pipeline-generator/tests/test_run_command.py b/tools/pipeline-generator/tests/test_run_command.py new file mode 100644 index 00000000..21fa503a --- /dev/null +++ b/tools/pipeline-generator/tests/test_run_command.py @@ -0,0 +1,583 @@ +# Copyright 2025 MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for the run command with validation fixes.""" + +import subprocess +from unittest.mock import Mock, patch + +from click.testing import CliRunner +from pipeline_generator.cli.run import _validate_results, run + + +class TestRunCommand: + """Test the run command functionality.""" + + def setup_method(self): + """Set up test fixtures.""" + self.runner = CliRunner() + + def test_run_missing_app_py(self, tmp_path): + """Test run command when app.py is missing.""" + app_path = tmp_path / "test_app" + app_path.mkdir() + input_dir = tmp_path / "input" + input_dir.mkdir() + output_dir = tmp_path / "output" + + # Create requirements.txt but not app.py + (app_path / "requirements.txt").write_text("monai-deploy-app-sdk\n") + + result = self.runner.invoke(run, [str(app_path), "--input", str(input_dir), "--output", str(output_dir)]) + + assert result.exit_code == 1 + assert "Error: app.py not found" in result.output + + def test_run_missing_requirements_txt(self, tmp_path): + """Test run command when requirements.txt is missing.""" + app_path = tmp_path / "test_app" + app_path.mkdir() + input_dir = tmp_path / "input" + input_dir.mkdir() + output_dir = tmp_path / "output" + + # Create app.py but not requirements.txt + (app_path / "app.py").write_text("print('test')") + + result = self.runner.invoke(run, [str(app_path), "--input", str(input_dir), "--output", str(output_dir)]) + + assert result.exit_code == 1 + assert "Error: requirements.txt not found" in result.output + + @patch("pipeline_generator.cli.run._validate_results") + @patch("subprocess.run") + @patch("subprocess.Popen") + def test_run_successful_with_new_venv(self, mock_popen, mock_run, mock_validate, tmp_path): + """Test successful run with new virtual environment creation.""" + # Set up test directories + app_path = tmp_path / "test_app" + app_path.mkdir() + input_dir = tmp_path / "input" + input_dir.mkdir() + output_dir = tmp_path / "output" + + # Create required files + (app_path / "app.py").write_text("print('test')") + (app_path / "requirements.txt").write_text("monai-deploy-app-sdk\n") + + # Mock subprocess for venv creation + mock_run.return_value = Mock(returncode=0) + + # Mock subprocess for app execution + mock_process = Mock() + mock_process.wait.return_value = 0 + mock_process.stdout = iter(["Processing...\n", "Complete!\n"]) + mock_popen.return_value = mock_process + + # Mock validation to return success + mock_validate.return_value = (True, "Generated 2 JSON files") + + result = self.runner.invoke(run, [str(app_path), "--input", str(input_dir), "--output", str(output_dir)]) + + assert result.exit_code == 0 + assert "Running MONAI Deploy application" in result.output + assert "Application completed successfully" in result.output + assert "Generated 2 JSON files" in result.output + mock_run.assert_called() # Verify venv was created + + @patch("pipeline_generator.cli.run._validate_results") + @patch("subprocess.run") + @patch("subprocess.Popen") + def test_run_skip_install(self, mock_popen, mock_run, mock_validate, tmp_path): + """Test run command with --skip-install flag.""" + # Set up test directories + app_path = tmp_path / "test_app" + app_path.mkdir() + input_dir = tmp_path / "input" + input_dir.mkdir() + output_dir = tmp_path / "output" + venv_path = app_path / ".venv" + venv_path.mkdir() + + # Create required files + (app_path / "app.py").write_text("print('test')") + (app_path / "requirements.txt").write_text("monai-deploy-app-sdk\n") + + # Mock subprocess for app execution + mock_process = Mock() + mock_process.wait.return_value = 0 + mock_process.stdout = iter(["Processing...\n", "Complete!\n"]) + mock_popen.return_value = mock_process + + # Mock validation to return success + mock_validate.return_value = (True, "Generated 1 JSON file") + + result = self.runner.invoke( + run, + [ + str(app_path), + "--input", + str(input_dir), + "--output", + str(output_dir), + "--skip-install", + ], + ) + + assert result.exit_code == 0 + assert "Running MONAI Deploy application" in result.output + mock_run.assert_not_called() # Verify no install happened + + @patch("pipeline_generator.cli.run._validate_results") + @patch("subprocess.run") + @patch("subprocess.Popen") + def test_run_with_model_path(self, mock_popen, mock_run, mock_validate, tmp_path): + """Test run command with custom model path.""" + # Set up test directories + app_path = tmp_path / "test_app" + app_path.mkdir() + input_dir = tmp_path / "input" + input_dir.mkdir() + output_dir = tmp_path / "output" + model_path = tmp_path / "custom_model" + model_path.mkdir() + venv_path = app_path / ".venv" + venv_path.mkdir() + + # Create required files + (app_path / "app.py").write_text("print('test')") + (app_path / "requirements.txt").write_text("monai-deploy-app-sdk\n") + + # Mock subprocess for app execution + mock_process = Mock() + mock_process.wait.return_value = 0 + mock_process.stdout = iter(["Processing...\n", "Complete!\n"]) + mock_popen.return_value = mock_process + + # Mock validation to return success + mock_validate.return_value = (True, "Generated 3 NIfTI files") + + result = self.runner.invoke( + run, + [ + str(app_path), + "--input", + str(input_dir), + "--output", + str(output_dir), + "--model", + str(model_path), + "--skip-install", + ], + ) + + assert result.exit_code == 0 + assert "Running MONAI Deploy application" in result.output + assert "Application completed successfully" in result.output + + def test_run_app_failure(self, tmp_path): + """Test run command when application fails.""" + app_path = tmp_path / "test_app" + app_path.mkdir() + input_dir = tmp_path / "input" + input_dir.mkdir() + output_dir = tmp_path / "output" + venv_path = app_path / ".venv" + venv_path.mkdir() + + # Create required files + (app_path / "app.py").write_text("import sys; sys.exit(1)") + (app_path / "requirements.txt").write_text("monai-deploy-app-sdk\n") + + with patch("subprocess.Popen") as mock_popen: + mock_process = Mock() + mock_process.wait.return_value = 1 # App fails + mock_process.stdout = iter(["Processing...\n", "Error!\n"]) + mock_popen.return_value = mock_process + + result = self.runner.invoke( + run, + [ + str(app_path), + "--input", + str(input_dir), + "--output", + str(output_dir), + "--skip-install", + ], + ) + + assert result.exit_code == 1 + assert "Application failed with exit code: 1" in result.output + + def test_run_venv_creation_failure(self, tmp_path): + """Test run command when virtual environment creation fails.""" + app_path = tmp_path / "test_app" + app_path.mkdir() + input_dir = tmp_path / "input" + input_dir.mkdir() + output_dir = tmp_path / "output" + + # Create required files + (app_path / "app.py").write_text("print('test')") + (app_path / "requirements.txt").write_text("monai-deploy-app-sdk\n") + + with patch("subprocess.run") as mock_run: + # Mock venv creation failure + mock_run.side_effect = subprocess.CalledProcessError(1, "python", stderr="venv creation failed") + + result = self.runner.invoke(run, [str(app_path), "--input", str(input_dir), "--output", str(output_dir)]) + + assert result.exit_code == 1 + assert "Error creating virtual environment" in result.output + + @patch("pipeline_generator.cli.run._validate_results") + @patch("subprocess.run") + @patch("subprocess.Popen") + def test_run_with_existing_venv(self, mock_popen, mock_run, mock_validate, tmp_path): + """Test run command with existing virtual environment.""" + # Set up test directories + app_path = tmp_path / "test_app" + app_path.mkdir() + input_dir = tmp_path / "input" + input_dir.mkdir() + output_dir = tmp_path / "output" + + # Create existing venv + venv_path = app_path / ".venv" + venv_path.mkdir() + (venv_path / "bin").mkdir() + (venv_path / "bin" / "python").touch() + (venv_path / "bin" / "pip").touch() + + # Create required files + (app_path / "app.py").write_text("print('test')") + (app_path / "requirements.txt").write_text("monai-deploy-app-sdk\n") + + # Mock subprocess for dependency installation + mock_run.return_value = Mock(returncode=0) + + # Mock subprocess for app execution + mock_process = Mock() + mock_process.wait.return_value = 0 + mock_process.stdout = iter(["Processing...\n", "Complete!\n"]) + mock_popen.return_value = mock_process + + # Mock validation to return success + mock_validate.return_value = (True, "Generated 1 image file") + + result = self.runner.invoke(run, [str(app_path), "--input", str(input_dir), "--output", str(output_dir)]) + + assert result.exit_code == 0 + assert "Using existing virtual environment" in result.output + assert "Application completed successfully" in result.output + + def test_run_pip_install_failure(self, tmp_path): + """Test run command when pip install fails.""" + app_path = tmp_path / "test_app" + app_path.mkdir() + input_dir = tmp_path / "input" + input_dir.mkdir() + output_dir = tmp_path / "output" + venv_path = app_path / ".venv" + venv_path.mkdir() + (venv_path / "bin").mkdir() + (venv_path / "bin" / "python").touch() + (venv_path / "bin" / "pip").touch() + + # Create required files + (app_path / "app.py").write_text("print('test')") + (app_path / "requirements.txt").write_text("nonexistent-package\n") + + with patch("subprocess.run") as mock_run: + # Mock pip install failure - need more calls due to local SDK installation + mock_run.side_effect = [ + Mock(returncode=0), # ensurepip success + Mock(returncode=0), # pip upgrade success + subprocess.CalledProcessError(1, "pip", stderr="package not found"), # local SDK install failure + subprocess.CalledProcessError(1, "pip", stderr="package not found"), # requirements install failure + ] + + result = self.runner.invoke(run, [str(app_path), "--input", str(input_dir), "--output", str(output_dir)]) + + assert result.exit_code == 1 + assert "Error installing dependencies" in result.output + + @patch("pipeline_generator.cli.run._validate_results") + @patch("subprocess.run") + @patch("subprocess.Popen") + def test_run_with_custom_venv_name(self, mock_popen, mock_run, mock_validate, tmp_path): + """Test run command with custom virtual environment name.""" + # Set up test directories + app_path = tmp_path / "test_app" + app_path.mkdir() + input_dir = tmp_path / "input" + input_dir.mkdir() + output_dir = tmp_path / "output" + + # Create required files + (app_path / "app.py").write_text("print('test')") + (app_path / "requirements.txt").write_text("monai-deploy-app-sdk\n") + + # Mock subprocess for venv creation + mock_run.return_value = Mock(returncode=0) + + # Mock subprocess for app execution + mock_process = Mock() + mock_process.wait.return_value = 0 + mock_process.stdout = iter(["Processing...\n", "Complete!\n"]) + mock_popen.return_value = mock_process + + # Mock validation to return success + mock_validate.return_value = (True, "Generated 4 JSON files") + + result = self.runner.invoke( + run, + [ + str(app_path), + "--input", + str(input_dir), + "--output", + str(output_dir), + "--venv-name", + "custom_venv", + ], + ) + + assert result.exit_code == 0 + assert "Running MONAI Deploy application" in result.output + assert "Application completed successfully" in result.output + + @patch("pipeline_generator.cli.run._validate_results") + @patch("subprocess.run") + @patch("subprocess.Popen") + def test_run_with_no_gpu(self, mock_popen, mock_run, mock_validate, tmp_path): + """Test run command with GPU disabled.""" + # Set up test directories + app_path = tmp_path / "test_app" + app_path.mkdir() + input_dir = tmp_path / "input" + input_dir.mkdir() + output_dir = tmp_path / "output" + venv_path = app_path / ".venv" + venv_path.mkdir() + + # Create required files + (app_path / "app.py").write_text("print('test')") + (app_path / "requirements.txt").write_text("monai-deploy-app-sdk\n") + + # Mock subprocess for app execution + mock_process = Mock() + mock_process.wait.return_value = 0 + mock_process.stdout = iter(["Processing...\n", "Complete!\n"]) + mock_popen.return_value = mock_process + + # Mock validation to return success + mock_validate.return_value = (True, "Generated 2 other files") + + result = self.runner.invoke( + run, + [ + str(app_path), + "--input", + str(input_dir), + "--output", + str(output_dir), + "--no-gpu", + "--skip-install", + ], + ) + + assert result.exit_code == 0 + assert "Running MONAI Deploy application" in result.output + assert "Application completed successfully" in result.output + + def test_validate_results_success(self, tmp_path): + """Test validation function with successful results.""" + output_dir = tmp_path / "output" + output_dir.mkdir() + + # Create test result files + (output_dir / "result1.json").write_text('{"test": "data"}') + (output_dir / "result2.json").write_text('{"test": "data2"}') + (output_dir / "image.png").write_text("fake image data") + + success, message = _validate_results(output_dir) + + assert success is True + assert "Generated 2 JSON files, 1 image file" in message + + def test_validate_results_no_files(self, tmp_path): + """Test validation function with no result files.""" + output_dir = tmp_path / "output" + output_dir.mkdir() + + success, message = _validate_results(output_dir) + + assert success is False + assert "No result files generated" in message + + def test_validate_results_missing_directory(self, tmp_path): + """Test validation function with missing output directory.""" + output_dir = tmp_path / "nonexistent" + + success, message = _validate_results(output_dir) + + assert success is False + assert "Output directory does not exist" in message + + @patch("pipeline_generator.cli.run._validate_results") + @patch("subprocess.run") + @patch("subprocess.Popen") + def test_run_validation_failure(self, mock_popen, mock_run, mock_validate, tmp_path): + """Test run command when validation fails.""" + # Set up test directories + app_path = tmp_path / "test_app" + app_path.mkdir() + input_dir = tmp_path / "input" + input_dir.mkdir() + output_dir = tmp_path / "output" + venv_path = app_path / ".venv" + venv_path.mkdir() + + # Create required files + (app_path / "app.py").write_text("print('test')") + (app_path / "requirements.txt").write_text("monai-deploy-app-sdk\n") + + # Mock subprocess for app execution (successful) + mock_process = Mock() + mock_process.wait.return_value = 0 + mock_process.stdout = iter(["Processing...\n", "Complete!\n"]) + mock_popen.return_value = mock_process + + # Mock validation to return failure + mock_validate.return_value = (False, "No result files generated") + + result = self.runner.invoke( + run, + [ + str(app_path), + "--input", + str(input_dir), + "--output", + str(output_dir), + "--skip-install", + ], + ) + + assert result.exit_code == 1 + assert "Application completed but failed validation" in result.output + assert "operator connection issues" in result.output + + def test_validate_results_nifti_files(self, tmp_path): + """Test validation function with NIfTI files.""" + output_dir = tmp_path / "output" + output_dir.mkdir() + + # Create test NIfTI files + (output_dir / "result1.nii").write_text("fake nifti data") + (output_dir / "result2.nii.gz").write_text("fake nifti data") + + success, message = _validate_results(output_dir) + + assert success is True + assert "Generated 2 NIfTI files" in message + + def test_validate_results_other_files(self, tmp_path): + """Test validation function with other file types.""" + output_dir = tmp_path / "output" + output_dir.mkdir() + + # Create test files of various types + (output_dir / "result.txt").write_text("text data") + (output_dir / "data.csv").write_text("csv data") + + success, message = _validate_results(output_dir) + + assert success is True + assert "Generated 2 other files" in message + + def test_validate_results_mixed_files(self, tmp_path): + """Test validation function with mixed file types.""" + output_dir = tmp_path / "output" + output_dir.mkdir() + + # Create test files of various types + (output_dir / "result.json").write_text('{"test": "data"}') + (output_dir / "image.png").write_text("fake png data") + (output_dir / "volume.nii").write_text("fake nifti data") + (output_dir / "report.txt").write_text("text report") + + success, message = _validate_results(output_dir) + + assert success is True + assert "1 JSON file" in message + assert "1 image file" in message + assert "1 NIfTI file" in message + assert "1 other file" in message + + @patch("pipeline_generator.cli.run._validate_results") + @patch("subprocess.run") + @patch("subprocess.Popen") + def test_run_keyboard_interrupt(self, mock_popen, mock_run, mock_validate, tmp_path): + """Test run command interrupted by user.""" + # Set up test directories + app_path = tmp_path / "test_app" + app_path.mkdir() + input_dir = tmp_path / "input" + input_dir.mkdir() + output_dir = tmp_path / "output" + venv_path = app_path / ".venv" + venv_path.mkdir() + + # Create required files + (app_path / "app.py").write_text("print('test')") + (app_path / "requirements.txt").write_text("monai-deploy-app-sdk\n") + + # Mock subprocess for app execution that raises KeyboardInterrupt + mock_process = Mock() + mock_process.stdout = iter(["Processing...\n"]) + mock_popen.return_value = mock_process + + # Simulate KeyboardInterrupt during execution + def mock_wait(): + raise KeyboardInterrupt("User interrupted") + + mock_process.wait = mock_wait + + result = self.runner.invoke( + run, + [ + str(app_path), + "--input", + str(input_dir), + "--output", + str(output_dir), + "--skip-install", + ], + ) + + assert result.exit_code == 1 + assert "Application interrupted by user" in result.output + + def test_main_execution(self): + """Test the main execution path.""" + # Test the main section logic + import pipeline_generator.cli.run as run_module + + # Mock the run function + with patch.object(run_module, "run") as mock_run: + # Simulate the __main__ execution by calling the main section directly + # This covers the: if __name__ == "__main__": run() line + if True: # Simulating __name__ == "__main__" + run_module.run() + + mock_run.assert_called_once() diff --git a/tools/pipeline-generator/tests/test_security.py b/tools/pipeline-generator/tests/test_security.py new file mode 100644 index 00000000..e15c326d --- /dev/null +++ b/tools/pipeline-generator/tests/test_security.py @@ -0,0 +1,117 @@ +# Copyright 2025 MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Test security features of the pipeline generator.""" + +from pathlib import Path + +import pytest +from pipeline_generator.generator.app_generator import AppGenerator + + +class TestSecurity: + """Test security measures in the app generator.""" + + def test_model_id_validation(self): + """Test that invalid model IDs are rejected.""" + generator = AppGenerator() + output_dir = Path("/tmp/test") + + # Valid model IDs + valid_ids = [ + "MONAI/spleen_ct_segmentation", + "test-org/model_name", + "user/model-with-dashes", + "org/model_with_underscores", + ] + + # Invalid model IDs that could cause code injection + invalid_ids = [ + "test; rm -rf /", # Shell command injection + "test' OR '1'='1", # SQL injection style + "test", # HTML/JS injection + "test`echo hacked`", # Command substitution + "test$(rm -rf /)", # Command substitution + "test\" + __import__('os').system('ls') + \"", # Python injection + "", # Empty + None, # None + ] + + # Test valid IDs (should not raise) + for model_id in valid_ids: + # We're just testing validation, not full generation + try: + # This will fail at download stage, but validation should pass + generator.generate_app(model_id, output_dir) + except ValueError as e: + if "Invalid model_id" in str(e): + pytest.fail(f"Valid model_id '{model_id!r}' was rejected: {e}") + # Other errors are fine (e.g., download failures) + + # Test invalid IDs (should raise ValueError) + for model_id in invalid_ids: + if model_id is None: + continue # Skip None test as it would fail at type checking + with pytest.raises(ValueError, match="Invalid model_id"): + generator.generate_app(model_id, output_dir) + + def test_app_name_sanitization(self): + """Test that app names are properly sanitized for Python identifiers.""" + # Test cases mapping input to expected sanitized output + test_cases = [ + ("test; rm -rf /", "test__rm__rfApp"), # Multiple special chars become underscores + ("test-with-dashes", "test_with_dashesApp"), + ("test.with.dots", "test_with_dotsApp"), + ("test space", "test_spaceApp"), + ("123test", "_123testApp"), # Starting with digit + ("Test", "TestApp"), # Normal case + ] + + for input_name, expected_class_name in test_cases: + # The AppGenerator will sanitize the name internally + # We test the sanitization function directly + sanitized = AppGenerator._sanitize_for_python_identifier(input_name) + result_with_app = f"{sanitized}App" + assert ( + result_with_app == expected_class_name + ), f"Failed for '{input_name!r}': got '{result_with_app!r}', expected '{expected_class_name!r}'" + + def test_sanitize_for_python_identifier(self): + """Test the Python identifier sanitization method.""" + test_cases = [ + ("normal_name", "normal_name"), + ("name-with-dashes", "name_with_dashes"), + ("name.with.dots", "name_with_dots"), + ("name with spaces", "name_with_spaces"), + ("123name", "_123name"), # Can't start with digit + ("", "app"), # Empty string + ("!@#$%", "app"), # All invalid chars + ("name!@#valid", "name___valid"), + ("CamelCase", "CamelCase"), # Preserve case + ] + + for input_str, expected in test_cases: + result = AppGenerator._sanitize_for_python_identifier(input_str) + assert result == expected, f"Failed for {input_str!r}: got {result!r}, expected {expected!r}" + + def test_no_autoescape_with_comment(self): + """Test that autoescape is disabled with proper documentation.""" + generator = AppGenerator() + + # Verify autoescape is False + assert generator.env.autoescape is False + + # The comment explaining why is in the source code + # This test just verifies the runtime behavior + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/tools/pipeline-generator/tests/test_settings.py b/tools/pipeline-generator/tests/test_settings.py new file mode 100644 index 00000000..e5fbad5a --- /dev/null +++ b/tools/pipeline-generator/tests/test_settings.py @@ -0,0 +1,120 @@ +# Copyright 2025 MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for settings and configuration.""" + +import tempfile +from pathlib import Path + +from pipeline_generator.config.settings import Endpoint, Settings, load_config + + +class TestEndpoint: + """Test Endpoint model.""" + + def test_endpoint_with_organization(self): + """Test creating endpoint with organization.""" + endpoint = Endpoint( + organization="MONAI", + base_url="https://huggingface.co", + description="MONAI models", + ) + + assert endpoint.organization == "MONAI" + assert endpoint.model_id is None + assert endpoint.base_url == "https://huggingface.co" + + def test_endpoint_with_model_id(self): + """Test creating endpoint with specific model ID.""" + endpoint = Endpoint(model_id="Project-MONAI/test", description="Test model") + + assert endpoint.organization is None + assert endpoint.model_id == "Project-MONAI/test" + assert endpoint.base_url == "https://huggingface.co" # default value + + +class TestSettings: + """Test Settings model.""" + + def test_empty_settings(self): + """Test creating empty settings.""" + settings = Settings() + + assert settings.endpoints == [] + assert settings.additional_models == [] + assert settings.get_all_endpoints() == [] + + def test_settings_with_endpoints(self): + """Test settings with endpoints.""" + endpoint1 = Endpoint(organization="MONAI") + endpoint2 = Endpoint(model_id="test/model") + + settings = Settings(endpoints=[endpoint1], additional_models=[endpoint2]) + + assert len(settings.endpoints) == 1 + assert len(settings.additional_models) == 1 + assert len(settings.get_all_endpoints()) == 2 + + def test_from_yaml(self): + """Test loading settings from YAML file.""" + yaml_content = """ +endpoints: + - organization: "MONAI" + base_url: "https://huggingface.co" + description: "Official MONAI models" + +additional_models: + - model_id: "Project-MONAI/test" + description: "Test model" +""" + + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + f.write(yaml_content) + f.flush() + + settings = Settings.from_yaml(Path(f.name)) + + assert len(settings.endpoints) == 1 + assert settings.endpoints[0].organization == "MONAI" + assert len(settings.additional_models) == 1 + assert settings.additional_models[0].model_id == "Project-MONAI/test" + + Path(f.name).unlink() + + +class TestLoadConfig: + """Test load_config function.""" + + def test_load_config_with_file(self): + """Test loading config from specified file.""" + yaml_content = """ +endpoints: + - organization: "TestOrg" +""" + + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + f.write(yaml_content) + f.flush() + + settings = load_config(Path(f.name)) + assert len(settings.endpoints) == 1 + assert settings.endpoints[0].organization == "TestOrg" + + Path(f.name).unlink() + + def test_load_config_default(self): + """Test loading config with default values when no file exists.""" + # Use a path that doesn't exist + settings = load_config(Path("/nonexistent/config.yaml")) + + assert len(settings.endpoints) == 1 + assert settings.endpoints[0].organization == "MONAI" + assert settings.endpoints[0].base_url == "https://huggingface.co" diff --git a/tools/pipeline-generator/tests/test_vlm_generation.py b/tools/pipeline-generator/tests/test_vlm_generation.py new file mode 100644 index 00000000..7c9a2388 --- /dev/null +++ b/tools/pipeline-generator/tests/test_vlm_generation.py @@ -0,0 +1,187 @@ +# Copyright 2025 MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for VLM model generation in pipeline generator.""" + +import tempfile +from pathlib import Path +from unittest.mock import patch + +import pytest + + +class TestVLMGeneration: + """Test VLM model generation functionality.""" + + @pytest.fixture + def temp_output_dir(self): + """Create temporary output directory.""" + with tempfile.TemporaryDirectory() as tmpdir: + yield Path(tmpdir) + + def test_vlm_config_identification(self): + """Test that custom input/output types are correctly identified.""" + from pipeline_generator.config.settings import load_config + + settings = load_config() + + # Find VLM model in config + vlm_models = [] + for endpoint in settings.endpoints: + for model in endpoint.models: + if model.input_type == "custom" and model.output_type == "custom": + vlm_models.append(model) + + # Should have at least the Llama3-VILA-M3-3B model + assert len(vlm_models) > 0 + assert any(m.model_id == "MONAI/Llama3-VILA-M3-3B" for m in vlm_models) + + def test_vlm_template_rendering(self, temp_output_dir): + """Test that VLM models use correct operators in template.""" + from jinja2 import Environment, FileSystemLoader + + # Set up template environment + template_dir = Path(__file__).parent.parent / "pipeline_generator" / "templates" + env = Environment( + loader=FileSystemLoader(str(template_dir)), + autoescape=False, + ) + + # Render template with VLM config + template = env.get_template("app.py.j2") + + # Test data for VLM model + context = { + "model_id": "MONAI/Llama3-VILA-M3-3B", + "app_name": "TestVLMApp", + "input_type": "custom", + "output_type": "custom", + "use_dicom": False, + "task": "Vision-Language Understanding", + "description": "Test VLM model", + "model_file": "model.safetensors", + "bundles": [], + "configs": [], + "preprocessing": {}, + "postprocessing": {}, + "output_postfix": "_pred", + "modality": "MR", + } + + rendered = template.render(**context) + + # Verify VLM operators are used + assert "PromptsLoaderOperator" in rendered + assert "Llama3VILAInferenceOperator" in rendered + assert "VLMResultsWriterOperator" in rendered + + # Verify standard operators are NOT used + assert "GenericDirectoryScanner" not in rendered + assert "NiftiDataLoader" not in rendered + assert "ImageFileLoader" not in rendered + assert "MonaiBundleInferenceOperator" not in rendered + + # Verify operator connections + assert "prompts_loader" in rendered + assert "vlm_inference" in rendered + assert "vlm_writer" in rendered + + # Verify port connections + assert '("prompt", "prompt")' in rendered + assert '("output_type", "output_type")' in rendered + assert '("request_id", "request_id")' in rendered + + def test_vlm_requirements_template(self): + """Test requirements.txt generation for VLM models.""" + from jinja2 import Environment, FileSystemLoader + + template_dir = Path(__file__).parent.parent / "pipeline_generator" / "templates" + env = Environment( + loader=FileSystemLoader(str(template_dir)), + autoescape=False, + ) + + template = env.get_template("requirements.txt.j2") + + context = { + "bundles": [], + "input_type": "custom", + "output_type": "custom", + "metadata": {}, + } + + rendered = template.render(**context) + + # Should include basic dependencies + assert "monai-deploy-app-sdk" in rendered.lower() + # VLM-specific deps are handled by operator optional imports + + def test_vlm_readme_template(self): + """Test README generation for VLM models.""" + from jinja2 import Environment, FileSystemLoader + + template_dir = Path(__file__).parent.parent / "pipeline_generator" / "templates" + env = Environment( + loader=FileSystemLoader(str(template_dir)), + autoescape=False, + ) + + template = env.get_template("README.md.j2") + + context = { + "model_id": "MONAI/Llama3-VILA-M3-3B", + "app_name": "Llama3VilaM33BApp", + "task": "Vision-Language Understanding", + "description": "VLM for medical image analysis", + "input_type": "custom", + "output_type": "custom", + "use_dicom": False, + "metadata": {"network_data_format": {"network": "Llama3-VILA-M3-3B"}}, + } + + rendered = template.render(**context) + + # Should mention VLM-specific usage + assert "MONAI/Llama3-VILA-M3-3B" in rendered + assert context["task"] in rendered + + @patch("pipeline_generator.core.hub_client.list_models") + def test_vlm_model_listing(self, mock_list_models): + """Test that VLM models appear correctly in listings.""" + from types import SimpleNamespace + + from pipeline_generator.core.hub_client import HuggingFaceClient + + # Mock the list_models response + mock_model = SimpleNamespace( + modelId="MONAI/Llama3-VILA-M3-3B", + tags=["medical", "vision-language"], + downloads=100, + likes=10, + name="Llama3-VILA-M3-3B", + author="MONAI", + description="VLM for medical imaging", + created_at=None, + lastModified=None, + siblings=[], + ) + + mock_list_models.return_value = [mock_model] + + client = HuggingFaceClient() + models = client.list_models_from_organization("MONAI") + + assert len(models) == 1 + assert models[0].model_id == "MONAI/Llama3-VILA-M3-3B" + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/tools/pipeline-generator/uv.lock b/tools/pipeline-generator/uv.lock new file mode 100644 index 00000000..cbde816e --- /dev/null +++ b/tools/pipeline-generator/uv.lock @@ -0,0 +1,587 @@ +version = 1 +revision = 3 +requires-python = "==3.10.*" + +[[package]] +name = "annotated-types" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ee/67/531ea369ba64dcff5ec9c3402f9f51bf748cec26dde048a2f973a4eea7f5/annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89", size = 16081, upload-time = "2024-05-20T21:33:25.928Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, +] + +[[package]] +name = "black" +version = "25.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "mypy-extensions" }, + { name = "packaging" }, + { name = "pathspec" }, + { name = "platformdirs" }, + { name = "tomli" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/94/49/26a7b0f3f35da4b5a65f081943b7bcd22d7002f5f0fb8098ec1ff21cb6ef/black-25.1.0.tar.gz", hash = "sha256:33496d5cd1222ad73391352b4ae8da15253c5de89b93a80b3e2c8d9a19ec2666", size = 649449, upload-time = "2025-01-29T04:15:40.373Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4d/3b/4ba3f93ac8d90410423fdd31d7541ada9bcee1df32fb90d26de41ed40e1d/black-25.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:759e7ec1e050a15f89b770cefbf91ebee8917aac5c20483bc2d80a6c3a04df32", size = 1629419, upload-time = "2025-01-29T05:37:06.642Z" }, + { url = "https://files.pythonhosted.org/packages/b4/02/0bde0485146a8a5e694daed47561785e8b77a0466ccc1f3e485d5ef2925e/black-25.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0e519ecf93120f34243e6b0054db49c00a35f84f195d5bce7e9f5cfc578fc2da", size = 1461080, upload-time = "2025-01-29T05:37:09.321Z" }, + { url = "https://files.pythonhosted.org/packages/52/0e/abdf75183c830eaca7589144ff96d49bce73d7ec6ad12ef62185cc0f79a2/black-25.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:055e59b198df7ac0b7efca5ad7ff2516bca343276c466be72eb04a3bcc1f82d7", size = 1766886, upload-time = "2025-01-29T04:18:24.432Z" }, + { url = "https://files.pythonhosted.org/packages/dc/a6/97d8bb65b1d8a41f8a6736222ba0a334db7b7b77b8023ab4568288f23973/black-25.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:db8ea9917d6f8fc62abd90d944920d95e73c83a5ee3383493e35d271aca872e9", size = 1419404, upload-time = "2025-01-29T04:19:04.296Z" }, + { url = "https://files.pythonhosted.org/packages/09/71/54e999902aed72baf26bca0d50781b01838251a462612966e9fc4891eadd/black-25.1.0-py3-none-any.whl", hash = "sha256:95e8176dae143ba9097f351d174fdaf0ccd29efb414b362ae3fd72bf0f710717", size = 207646, upload-time = "2025-01-29T04:15:38.082Z" }, +] + +[[package]] +name = "certifi" +version = "2025.8.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/dc/67/960ebe6bf230a96cda2e0abcf73af550ec4f090005363542f0765df162e0/certifi-2025.8.3.tar.gz", hash = "sha256:e564105f78ded564e3ae7c923924435e1daa7463faeab5bb932bc53ffae63407", size = 162386, upload-time = "2025-08-03T03:07:47.08Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/48/1549795ba7742c948d2ad169c1c8cdbae65bc450d6cd753d124b17c8cd32/certifi-2025.8.3-py3-none-any.whl", hash = "sha256:f6c12493cfb1b06ba2ff328595af9350c65d6644968e5d3a2ffd78699af217a5", size = 161216, upload-time = "2025-08-03T03:07:45.777Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e4/33/89c2ced2b67d1c2a61c19c6751aa8902d46ce3dacb23600a283619f5a12d/charset_normalizer-3.4.2.tar.gz", hash = "sha256:5baececa9ecba31eff645232d59845c07aa030f0c81ee70184a90d35099a0e63", size = 126367, upload-time = "2025-05-02T08:34:42.01Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/95/28/9901804da60055b406e1a1c5ba7aac1276fb77f1dde635aabfc7fd84b8ab/charset_normalizer-3.4.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7c48ed483eb946e6c04ccbe02c6b4d1d48e51944b6db70f697e089c193404941", size = 201818, upload-time = "2025-05-02T08:31:46.725Z" }, + { url = "https://files.pythonhosted.org/packages/d9/9b/892a8c8af9110935e5adcbb06d9c6fe741b6bb02608c6513983048ba1a18/charset_normalizer-3.4.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b2d318c11350e10662026ad0eb71bb51c7812fc8590825304ae0bdd4ac283acd", size = 144649, upload-time = "2025-05-02T08:31:48.889Z" }, + { url = "https://files.pythonhosted.org/packages/7b/a5/4179abd063ff6414223575e008593861d62abfc22455b5d1a44995b7c101/charset_normalizer-3.4.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9cbfacf36cb0ec2897ce0ebc5d08ca44213af24265bd56eca54bee7923c48fd6", size = 155045, upload-time = "2025-05-02T08:31:50.757Z" }, + { url = "https://files.pythonhosted.org/packages/3b/95/bc08c7dfeddd26b4be8c8287b9bb055716f31077c8b0ea1cd09553794665/charset_normalizer-3.4.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:18dd2e350387c87dabe711b86f83c9c78af772c748904d372ade190b5c7c9d4d", size = 147356, upload-time = "2025-05-02T08:31:52.634Z" }, + { url = "https://files.pythonhosted.org/packages/a8/2d/7a5b635aa65284bf3eab7653e8b4151ab420ecbae918d3e359d1947b4d61/charset_normalizer-3.4.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8075c35cd58273fee266c58c0c9b670947c19df5fb98e7b66710e04ad4e9ff86", size = 149471, upload-time = "2025-05-02T08:31:56.207Z" }, + { url = "https://files.pythonhosted.org/packages/ae/38/51fc6ac74251fd331a8cfdb7ec57beba8c23fd5493f1050f71c87ef77ed0/charset_normalizer-3.4.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5bf4545e3b962767e5c06fe1738f951f77d27967cb2caa64c28be7c4563e162c", size = 151317, upload-time = "2025-05-02T08:31:57.613Z" }, + { url = "https://files.pythonhosted.org/packages/b7/17/edee1e32215ee6e9e46c3e482645b46575a44a2d72c7dfd49e49f60ce6bf/charset_normalizer-3.4.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:7a6ab32f7210554a96cd9e33abe3ddd86732beeafc7a28e9955cdf22ffadbab0", size = 146368, upload-time = "2025-05-02T08:31:59.468Z" }, + { url = "https://files.pythonhosted.org/packages/26/2c/ea3e66f2b5f21fd00b2825c94cafb8c326ea6240cd80a91eb09e4a285830/charset_normalizer-3.4.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:b33de11b92e9f75a2b545d6e9b6f37e398d86c3e9e9653c4864eb7e89c5773ef", size = 154491, upload-time = "2025-05-02T08:32:01.219Z" }, + { url = "https://files.pythonhosted.org/packages/52/47/7be7fa972422ad062e909fd62460d45c3ef4c141805b7078dbab15904ff7/charset_normalizer-3.4.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:8755483f3c00d6c9a77f490c17e6ab0c8729e39e6390328e42521ef175380ae6", size = 157695, upload-time = "2025-05-02T08:32:03.045Z" }, + { url = "https://files.pythonhosted.org/packages/2f/42/9f02c194da282b2b340f28e5fb60762de1151387a36842a92b533685c61e/charset_normalizer-3.4.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:68a328e5f55ec37c57f19ebb1fdc56a248db2e3e9ad769919a58672958e8f366", size = 154849, upload-time = "2025-05-02T08:32:04.651Z" }, + { url = "https://files.pythonhosted.org/packages/67/44/89cacd6628f31fb0b63201a618049be4be2a7435a31b55b5eb1c3674547a/charset_normalizer-3.4.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:21b2899062867b0e1fde9b724f8aecb1af14f2778d69aacd1a5a1853a597a5db", size = 150091, upload-time = "2025-05-02T08:32:06.719Z" }, + { url = "https://files.pythonhosted.org/packages/1f/79/4b8da9f712bc079c0f16b6d67b099b0b8d808c2292c937f267d816ec5ecc/charset_normalizer-3.4.2-cp310-cp310-win32.whl", hash = "sha256:e8082b26888e2f8b36a042a58307d5b917ef2b1cacab921ad3323ef91901c71a", size = 98445, upload-time = "2025-05-02T08:32:08.66Z" }, + { url = "https://files.pythonhosted.org/packages/7d/d7/96970afb4fb66497a40761cdf7bd4f6fca0fc7bafde3a84f836c1f57a926/charset_normalizer-3.4.2-cp310-cp310-win_amd64.whl", hash = "sha256:f69a27e45c43520f5487f27627059b64aaf160415589230992cec34c5e18a509", size = 105782, upload-time = "2025-05-02T08:32:10.46Z" }, + { url = "https://files.pythonhosted.org/packages/20/94/c5790835a017658cbfabd07f3bfb549140c3ac458cfc196323996b10095a/charset_normalizer-3.4.2-py3-none-any.whl", hash = "sha256:7f56930ab0abd1c45cd15be65cc741c28b1c9a34876ce8c17a2fa107810c0af0", size = 52626, upload-time = "2025-05-02T08:34:40.053Z" }, +] + +[[package]] +name = "click" +version = "8.2.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/60/6c/8ca2efa64cf75a977a0d7fac081354553ebe483345c734fb6b6515d96bbc/click-8.2.1.tar.gz", hash = "sha256:27c491cc05d968d271d5a1db13e3b5a184636d9d930f148c50b038f0d0646202", size = 286342, upload-time = "2025-05-20T23:19:49.832Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/85/32/10bb5764d90a8eee674e9dc6f4db6a0ab47c8c4d0d83c27f7c39ac415a4d/click-8.2.1-py3-none-any.whl", hash = "sha256:61a3265b914e850b85317d0b3109c7f8cd35a670f963866005d6ef1d5175a12b", size = 102215, upload-time = "2025-05-20T23:19:47.796Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "coverage" +version = "7.10.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ee/76/17780846fc7aade1e66712e1e27dd28faa0a5d987a1f433610974959eaa8/coverage-7.10.2.tar.gz", hash = "sha256:5d6e6d84e6dd31a8ded64759626627247d676a23c1b892e1326f7c55c8d61055", size = 820754, upload-time = "2025-08-04T00:35:17.511Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d8/5f/5ce748ab3f142593698aff5f8a0cf020775aa4e24b9d8748b5a56b64d3f8/coverage-7.10.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:79f0283ab5e6499fd5fe382ca3d62afa40fb50ff227676a3125d18af70eabf65", size = 215003, upload-time = "2025-08-04T00:33:02.977Z" }, + { url = "https://files.pythonhosted.org/packages/f4/ed/507088561217b000109552139802fa99c33c16ad19999c687b601b3790d0/coverage-7.10.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4545e906f595ee8ab8e03e21be20d899bfc06647925bc5b224ad7e8c40e08b8", size = 215391, upload-time = "2025-08-04T00:33:05.645Z" }, + { url = "https://files.pythonhosted.org/packages/79/1b/0f496259fe137c4c5e1e8eaff496fb95af88b71700f5e57725a4ddbe742b/coverage-7.10.2-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:ae385e1d58fbc6a9b1c315e5510ac52281e271478b45f92ca9b5ad42cf39643f", size = 242367, upload-time = "2025-08-04T00:33:07.189Z" }, + { url = "https://files.pythonhosted.org/packages/b9/8e/5a8835fb0122a2e2a108bf3527931693c4625fdc4d953950a480b9625852/coverage-7.10.2-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6f0cbe5f7dd19f3a32bac2251b95d51c3b89621ac88a2648096ce40f9a5aa1e7", size = 243627, upload-time = "2025-08-04T00:33:08.809Z" }, + { url = "https://files.pythonhosted.org/packages/c3/96/6a528429c2e0e8d85261764d0cd42e51a429510509bcc14676ee5d1bb212/coverage-7.10.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fd17f427f041f6b116dc90b4049c6f3e1230524407d00daa2d8c7915037b5947", size = 245485, upload-time = "2025-08-04T00:33:10.29Z" }, + { url = "https://files.pythonhosted.org/packages/bf/82/1fba935c4d02c33275aca319deabf1f22c0f95f2c0000bf7c5f276d6f7b4/coverage-7.10.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:7f10ca4cde7b466405cce0a0e9971a13eb22e57a5ecc8b5f93a81090cc9c7eb9", size = 243429, upload-time = "2025-08-04T00:33:11.909Z" }, + { url = "https://files.pythonhosted.org/packages/fc/a8/c8dc0a57a729fc93be33ab78f187a8f52d455fa8f79bfb379fe23b45868d/coverage-7.10.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:3b990df23dd51dccce26d18fb09fd85a77ebe46368f387b0ffba7a74e470b31b", size = 242104, upload-time = "2025-08-04T00:33:13.467Z" }, + { url = "https://files.pythonhosted.org/packages/b9/6f/0b7da1682e2557caeed299a00897b42afde99a241a01eba0197eb982b90f/coverage-7.10.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:cc3902584d25c7eef57fb38f440aa849a26a3a9f761a029a72b69acfca4e31f8", size = 242397, upload-time = "2025-08-04T00:33:14.682Z" }, + { url = "https://files.pythonhosted.org/packages/2d/e4/54dc833dadccd519c04a28852f39a37e522bad35d70cfe038817cdb8f168/coverage-7.10.2-cp310-cp310-win32.whl", hash = "sha256:9dd37e9ac00d5eb72f38ed93e3cdf2280b1dbda3bb9b48c6941805f265ad8d87", size = 217502, upload-time = "2025-08-04T00:33:16.254Z" }, + { url = "https://files.pythonhosted.org/packages/c3/e7/2f78159c4c127549172f427dff15b02176329327bf6a6a1fcf1f603b5456/coverage-7.10.2-cp310-cp310-win_amd64.whl", hash = "sha256:99d16f15cb5baf0729354c5bd3080ae53847a4072b9ba1e10957522fb290417f", size = 218388, upload-time = "2025-08-04T00:33:17.4Z" }, + { url = "https://files.pythonhosted.org/packages/18/d8/9b768ac73a8ac2d10c080af23937212434a958c8d2a1c84e89b450237942/coverage-7.10.2-py3-none-any.whl", hash = "sha256:95db3750dd2e6e93d99fa2498f3a1580581e49c494bddccc6f85c5c21604921f", size = 206973, upload-time = "2025-08-04T00:35:15.918Z" }, +] + +[package.optional-dependencies] +toml = [ + { name = "tomli" }, +] + +[[package]] +name = "exceptiongroup" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0b/9f/a65090624ecf468cdca03533906e7c69ed7588582240cfe7cc9e770b50eb/exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88", size = 29749, upload-time = "2025-05-10T17:42:51.123Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/36/f4/c6e662dade71f56cd2f3735141b265c3c79293c109549c1e6933b0651ffc/exceptiongroup-1.3.0-py3-none-any.whl", hash = "sha256:4d111e6e0c13d0644cad6ddaa7ed0261a0b36971f6d23e7ec9b4b9097da78a10", size = 16674, upload-time = "2025-05-10T17:42:49.33Z" }, +] + +[[package]] +name = "filelock" +version = "3.18.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0a/10/c23352565a6544bdc5353e0b15fc1c563352101f30e24bf500207a54df9a/filelock-3.18.0.tar.gz", hash = "sha256:adbc88eabb99d2fec8c9c1b229b171f18afa655400173ddc653d5d01501fb9f2", size = 18075, upload-time = "2025-03-14T07:11:40.47Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4d/36/2a115987e2d8c300a974597416d9de88f2444426de9571f4b59b2cca3acc/filelock-3.18.0-py3-none-any.whl", hash = "sha256:c401f4f8377c4464e6db25fff06205fd89bdd83b65eb0488ed1b160f780e21de", size = 16215, upload-time = "2025-03-14T07:11:39.145Z" }, +] + +[[package]] +name = "flake8" +version = "7.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mccabe" }, + { name = "pycodestyle" }, + { name = "pyflakes" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9b/af/fbfe3c4b5a657d79e5c47a2827a362f9e1b763336a52f926126aa6dc7123/flake8-7.3.0.tar.gz", hash = "sha256:fe044858146b9fc69b551a4b490d69cf960fcb78ad1edcb84e7fbb1b4a8e3872", size = 48326, upload-time = "2025-06-20T19:31:35.838Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9f/56/13ab06b4f93ca7cac71078fbe37fcea175d3216f31f85c3168a6bbd0bb9a/flake8-7.3.0-py2.py3-none-any.whl", hash = "sha256:b9696257b9ce8beb888cdbe31cf885c90d31928fe202be0889a7cdafad32f01e", size = 57922, upload-time = "2025-06-20T19:31:34.425Z" }, +] + +[[package]] +name = "fsspec" +version = "2025.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8b/02/0835e6ab9cfc03916fe3f78c0956cfcdb6ff2669ffa6651065d5ebf7fc98/fsspec-2025.7.0.tar.gz", hash = "sha256:786120687ffa54b8283d942929540d8bc5ccfa820deb555a2b5d0ed2b737bf58", size = 304432, upload-time = "2025-07-15T16:05:21.19Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2f/e0/014d5d9d7a4564cf1c40b5039bc882db69fd881111e03ab3657ac0b218e2/fsspec-2025.7.0-py3-none-any.whl", hash = "sha256:8b012e39f63c7d5f10474de957f3ab793b47b45ae7d39f2fb735f8bbe25c0e21", size = 199597, upload-time = "2025-07-15T16:05:19.529Z" }, +] + +[[package]] +name = "hf-xet" +version = "1.1.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b2/0a/a0f56735940fde6dd627602fec9ab3bad23f66a272397560abd65aba416e/hf_xet-1.1.7.tar.gz", hash = "sha256:20cec8db4561338824a3b5f8c19774055b04a8df7fff0cb1ff2cb1a0c1607b80", size = 477719, upload-time = "2025-08-06T00:30:55.741Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b1/7c/8d7803995caf14e7d19a392a486a040f923e2cfeff824e9b800b92072f76/hf_xet-1.1.7-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:60dae4b44d520819e54e216a2505685248ec0adbdb2dd4848b17aa85a0375cde", size = 2761743, upload-time = "2025-08-06T00:30:50.634Z" }, + { url = "https://files.pythonhosted.org/packages/51/a3/fa5897099454aa287022a34a30e68dbff0e617760f774f8bd1db17f06bd4/hf_xet-1.1.7-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:b109f4c11e01c057fc82004c9e51e6cdfe2cb230637644ade40c599739067b2e", size = 2624331, upload-time = "2025-08-06T00:30:49.212Z" }, + { url = "https://files.pythonhosted.org/packages/86/50/2446a132267e60b8a48b2e5835d6e24fd988000d0f5b9b15ebd6d64ef769/hf_xet-1.1.7-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6efaaf1a5a9fc3a501d3e71e88a6bfebc69ee3a716d0e713a931c8b8d920038f", size = 3183844, upload-time = "2025-08-06T00:30:47.582Z" }, + { url = "https://files.pythonhosted.org/packages/20/8f/ccc670616bb9beee867c6bb7139f7eab2b1370fe426503c25f5cbb27b148/hf_xet-1.1.7-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:751571540f9c1fbad9afcf222a5fb96daf2384bf821317b8bfb0c59d86078513", size = 3074209, upload-time = "2025-08-06T00:30:45.509Z" }, + { url = "https://files.pythonhosted.org/packages/21/0a/4c30e1eb77205565b854f5e4a82cf1f056214e4dc87f2918ebf83d47ae14/hf_xet-1.1.7-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:18b61bbae92d56ae731b92087c44efcac216071182c603fc535f8e29ec4b09b8", size = 3239602, upload-time = "2025-08-06T00:30:52.41Z" }, + { url = "https://files.pythonhosted.org/packages/f5/1e/fc7e9baf14152662ef0b35fa52a6e889f770a7ed14ac239de3c829ecb47e/hf_xet-1.1.7-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:713f2bff61b252f8523739969f247aa354ad8e6d869b8281e174e2ea1bb8d604", size = 3348184, upload-time = "2025-08-06T00:30:54.105Z" }, + { url = "https://files.pythonhosted.org/packages/a3/73/e354eae84ceff117ec3560141224724794828927fcc013c5b449bf0b8745/hf_xet-1.1.7-cp37-abi3-win_amd64.whl", hash = "sha256:2e356da7d284479ae0f1dea3cf5a2f74fdf925d6dca84ac4341930d892c7cb34", size = 2820008, upload-time = "2025-08-06T00:30:57.056Z" }, +] + +[[package]] +name = "huggingface-hub" +version = "0.34.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "filelock" }, + { name = "fsspec" }, + { name = "hf-xet", marker = "platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" }, + { name = "packaging" }, + { name = "pyyaml" }, + { name = "requests" }, + { name = "tqdm" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/91/b4/e6b465eca5386b52cf23cb6df8644ad318a6b0e12b4b96a7e0be09cbfbcc/huggingface_hub-0.34.3.tar.gz", hash = "sha256:d58130fd5aa7408480681475491c0abd7e835442082fbc3ef4d45b6c39f83853", size = 456800, upload-time = "2025-07-29T08:38:53.885Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/59/a8/4677014e771ed1591a87b63a2392ce6923baf807193deef302dcfde17542/huggingface_hub-0.34.3-py3-none-any.whl", hash = "sha256:5444550099e2d86e68b2898b09e85878fbd788fc2957b506c6a79ce060e39492", size = 558847, upload-time = "2025-07-29T08:38:51.904Z" }, +] + +[[package]] +name = "idna" +version = "3.10" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f1/70/7703c29685631f5a7590aa73f1f1d3fa9a380e654b86af429e0934a32f7d/idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9", size = 190490, upload-time = "2024-09-15T18:07:39.745Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", size = 70442, upload-time = "2024-09-15T18:07:37.964Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f2/97/ebf4da567aa6827c909642694d71c9fcf53e5b504f2d96afea02718862f3/iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7", size = 4793, upload-time = "2025-03-19T20:09:59.721Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760", size = 6050, upload-time = "2025-03-19T20:10:01.071Z" }, +] + +[[package]] +name = "jinja2" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, +] + +[[package]] +name = "markdown-it-py" +version = "3.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mdurl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/38/71/3b932df36c1a044d397a1f92d1cf91ee0a503d91e470cbd670aa66b07ed0/markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb", size = 74596, upload-time = "2023-06-03T06:41:14.443Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/42/d7/1ec15b46af6af88f19b8e5ffea08fa375d433c998b8a7639e76935c14f1f/markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1", size = 87528, upload-time = "2023-06-03T06:41:11.019Z" }, +] + +[[package]] +name = "markupsafe" +version = "3.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b2/97/5d42485e71dfc078108a86d6de8fa46db44a1a9295e89c5d6d4a06e23a62/markupsafe-3.0.2.tar.gz", hash = "sha256:ee55d3edf80167e48ea11a923c7386f4669df67d7994554387f84e7d8b0a2bf0", size = 20537, upload-time = "2024-10-18T15:21:54.129Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/90/d08277ce111dd22f77149fd1a5d4653eeb3b3eaacbdfcbae5afb2600eebd/MarkupSafe-3.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7e94c425039cde14257288fd61dcfb01963e658efbc0ff54f5306b06054700f8", size = 14357, upload-time = "2024-10-18T15:20:51.44Z" }, + { url = "https://files.pythonhosted.org/packages/04/e1/6e2194baeae0bca1fae6629dc0cbbb968d4d941469cbab11a3872edff374/MarkupSafe-3.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9e2d922824181480953426608b81967de705c3cef4d1af983af849d7bd619158", size = 12393, upload-time = "2024-10-18T15:20:52.426Z" }, + { url = "https://files.pythonhosted.org/packages/1d/69/35fa85a8ece0a437493dc61ce0bb6d459dcba482c34197e3efc829aa357f/MarkupSafe-3.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:38a9ef736c01fccdd6600705b09dc574584b89bea478200c5fbf112a6b0d5579", size = 21732, upload-time = "2024-10-18T15:20:53.578Z" }, + { url = "https://files.pythonhosted.org/packages/22/35/137da042dfb4720b638d2937c38a9c2df83fe32d20e8c8f3185dbfef05f7/MarkupSafe-3.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bbcb445fa71794da8f178f0f6d66789a28d7319071af7a496d4d507ed566270d", size = 20866, upload-time = "2024-10-18T15:20:55.06Z" }, + { url = "https://files.pythonhosted.org/packages/29/28/6d029a903727a1b62edb51863232152fd335d602def598dade38996887f0/MarkupSafe-3.0.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:57cb5a3cf367aeb1d316576250f65edec5bb3be939e9247ae594b4bcbc317dfb", size = 20964, upload-time = "2024-10-18T15:20:55.906Z" }, + { url = "https://files.pythonhosted.org/packages/cc/cd/07438f95f83e8bc028279909d9c9bd39e24149b0d60053a97b2bc4f8aa51/MarkupSafe-3.0.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:3809ede931876f5b2ec92eef964286840ed3540dadf803dd570c3b7e13141a3b", size = 21977, upload-time = "2024-10-18T15:20:57.189Z" }, + { url = "https://files.pythonhosted.org/packages/29/01/84b57395b4cc062f9c4c55ce0df7d3108ca32397299d9df00fedd9117d3d/MarkupSafe-3.0.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e07c3764494e3776c602c1e78e298937c3315ccc9043ead7e685b7f2b8d47b3c", size = 21366, upload-time = "2024-10-18T15:20:58.235Z" }, + { url = "https://files.pythonhosted.org/packages/bd/6e/61ebf08d8940553afff20d1fb1ba7294b6f8d279df9fd0c0db911b4bbcfd/MarkupSafe-3.0.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b424c77b206d63d500bcb69fa55ed8d0e6a3774056bdc4839fc9298a7edca171", size = 21091, upload-time = "2024-10-18T15:20:59.235Z" }, + { url = "https://files.pythonhosted.org/packages/11/23/ffbf53694e8c94ebd1e7e491de185124277964344733c45481f32ede2499/MarkupSafe-3.0.2-cp310-cp310-win32.whl", hash = "sha256:fcabf5ff6eea076f859677f5f0b6b5c1a51e70a376b0579e0eadef8db48c6b50", size = 15065, upload-time = "2024-10-18T15:21:00.307Z" }, + { url = "https://files.pythonhosted.org/packages/44/06/e7175d06dd6e9172d4a69a72592cb3f7a996a9c396eee29082826449bbc3/MarkupSafe-3.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:6af100e168aa82a50e186c82875a5893c5597a0c1ccdb0d8b40240b1f28b969a", size = 15514, upload-time = "2024-10-18T15:21:01.122Z" }, +] + +[[package]] +name = "mccabe" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e7/ff/0ffefdcac38932a54d2b5eed4e0ba8a408f215002cd178ad1df0f2806ff8/mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325", size = 9658, upload-time = "2022-01-24T01:14:51.113Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/1a/1f68f9ba0c207934b35b86a8ca3aad8395a3d6dd7921c0686e23853ff5a9/mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e", size = 7350, upload-time = "2022-01-24T01:14:49.62Z" }, +] + +[[package]] +name = "mdurl" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, +] + +[[package]] +name = "mypy" +version = "1.17.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mypy-extensions" }, + { name = "pathspec" }, + { name = "tomli" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8e/22/ea637422dedf0bf36f3ef238eab4e455e2a0dcc3082b5cc067615347ab8e/mypy-1.17.1.tar.gz", hash = "sha256:25e01ec741ab5bb3eec8ba9cdb0f769230368a22c959c4937360efb89b7e9f01", size = 3352570, upload-time = "2025-07-31T07:54:19.204Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/77/a9/3d7aa83955617cdf02f94e50aab5c830d205cfa4320cf124ff64acce3a8e/mypy-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3fbe6d5555bf608c47203baa3e72dbc6ec9965b3d7c318aa9a4ca76f465bd972", size = 11003299, upload-time = "2025-07-31T07:54:06.425Z" }, + { url = "https://files.pythonhosted.org/packages/83/e8/72e62ff837dd5caaac2b4a5c07ce769c8e808a00a65e5d8f94ea9c6f20ab/mypy-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:80ef5c058b7bce08c83cac668158cb7edea692e458d21098c7d3bce35a5d43e7", size = 10125451, upload-time = "2025-07-31T07:53:52.974Z" }, + { url = "https://files.pythonhosted.org/packages/7d/10/f3f3543f6448db11881776f26a0ed079865926b0c841818ee22de2c6bbab/mypy-1.17.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c4a580f8a70c69e4a75587bd925d298434057fe2a428faaf927ffe6e4b9a98df", size = 11916211, upload-time = "2025-07-31T07:53:18.879Z" }, + { url = "https://files.pythonhosted.org/packages/06/bf/63e83ed551282d67bb3f7fea2cd5561b08d2bb6eb287c096539feb5ddbc5/mypy-1.17.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dd86bb649299f09d987a2eebb4d52d10603224500792e1bee18303bbcc1ce390", size = 12652687, upload-time = "2025-07-31T07:53:30.544Z" }, + { url = "https://files.pythonhosted.org/packages/69/66/68f2eeef11facf597143e85b694a161868b3b006a5fbad50e09ea117ef24/mypy-1.17.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:a76906f26bd8d51ea9504966a9c25419f2e668f012e0bdf3da4ea1526c534d94", size = 12896322, upload-time = "2025-07-31T07:53:50.74Z" }, + { url = "https://files.pythonhosted.org/packages/a3/87/8e3e9c2c8bd0d7e071a89c71be28ad088aaecbadf0454f46a540bda7bca6/mypy-1.17.1-cp310-cp310-win_amd64.whl", hash = "sha256:e79311f2d904ccb59787477b7bd5d26f3347789c06fcd7656fa500875290264b", size = 9507962, upload-time = "2025-07-31T07:53:08.431Z" }, + { url = "https://files.pythonhosted.org/packages/1d/f3/8fcd2af0f5b806f6cf463efaffd3c9548a28f84220493ecd38d127b6b66d/mypy-1.17.1-py3-none-any.whl", hash = "sha256:a9f52c0351c21fe24c21d8c0eb1f62967b262d6729393397b6f443c3b773c3b9", size = 2283411, upload-time = "2025-07-31T07:53:24.664Z" }, +] + +[[package]] +name = "mypy-extensions" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, +] + +[[package]] +name = "packaging" +version = "25.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727, upload-time = "2025-04-19T11:48:59.673Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, +] + +[[package]] +name = "pathspec" +version = "0.12.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ca/bc/f35b8446f4531a7cb215605d100cd88b7ac6f44ab3fc94870c120ab3adbf/pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712", size = 51043, upload-time = "2023-12-10T22:30:45Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cc/20/ff623b09d963f88bfde16306a54e12ee5ea43e9b597108672ff3a408aad6/pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", size = 31191, upload-time = "2023-12-10T22:30:43.14Z" }, +] + +[[package]] +name = "pipeline-generator" +version = "1.0.0" +source = { editable = "." } +dependencies = [ + { name = "click" }, + { name = "huggingface-hub" }, + { name = "jinja2" }, + { name = "pydantic" }, + { name = "pyyaml" }, + { name = "rich" }, +] + +[package.dev-dependencies] +dev = [ + { name = "black" }, + { name = "flake8" }, + { name = "mypy" }, + { name = "pytest" }, + { name = "pytest-cov" }, + { name = "types-pyyaml" }, +] + +[package.metadata] +requires-dist = [ + { name = "click", specifier = ">=8.2.1" }, + { name = "huggingface-hub", specifier = ">=0.34.3" }, + { name = "jinja2", specifier = ">=3.1.6" }, + { name = "pydantic", specifier = ">=2.11.7" }, + { name = "pyyaml", specifier = ">=6.0.2" }, + { name = "rich", specifier = ">=14.1.0" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "black", specifier = ">=25.1.0" }, + { name = "flake8", specifier = ">=7.3.0" }, + { name = "mypy", specifier = ">=1.17.1" }, + { name = "pytest", specifier = ">=8.4.1" }, + { name = "pytest-cov", specifier = ">=6.2.1" }, + { name = "types-pyyaml", specifier = ">=6.0.12.20250516" }, +] + +[[package]] +name = "platformdirs" +version = "4.3.8" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fe/8b/3c73abc9c759ecd3f1f7ceff6685840859e8070c4d947c93fae71f6a0bf2/platformdirs-4.3.8.tar.gz", hash = "sha256:3d512d96e16bcb959a814c9f348431070822a6496326a4be0911c40b5a74c2bc", size = 21362, upload-time = "2025-05-07T22:47:42.121Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fe/39/979e8e21520d4e47a0bbe349e2713c0aac6f3d853d0e5b34d76206c439aa/platformdirs-4.3.8-py3-none-any.whl", hash = "sha256:ff7059bb7eb1179e2685604f4aaf157cfd9535242bd23742eadc3c13542139b4", size = 18567, upload-time = "2025-05-07T22:47:40.376Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pycodestyle" +version = "2.14.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/11/e0/abfd2a0d2efe47670df87f3e3a0e2edda42f055053c85361f19c0e2c1ca8/pycodestyle-2.14.0.tar.gz", hash = "sha256:c4b5b517d278089ff9d0abdec919cd97262a3367449ea1c8b49b91529167b783", size = 39472, upload-time = "2025-06-20T18:49:48.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d7/27/a58ddaf8c588a3ef080db9d0b7e0b97215cee3a45df74f3a94dbbf5c893a/pycodestyle-2.14.0-py2.py3-none-any.whl", hash = "sha256:dd6bf7cb4ee77f8e016f9c8e74a35ddd9f67e1d5fd4184d86c3b98e07099f42d", size = 31594, upload-time = "2025-06-20T18:49:47.491Z" }, +] + +[[package]] +name = "pydantic" +version = "2.11.7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/00/dd/4325abf92c39ba8623b5af936ddb36ffcfe0beae70405d456ab1fb2f5b8c/pydantic-2.11.7.tar.gz", hash = "sha256:d989c3c6cb79469287b1569f7447a17848c998458d49ebe294e975b9baf0f0db", size = 788350, upload-time = "2025-06-14T08:33:17.137Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6a/c0/ec2b1c8712ca690e5d61979dee872603e92b8a32f94cc1b72d53beab008a/pydantic-2.11.7-py3-none-any.whl", hash = "sha256:dde5df002701f6de26248661f6835bbe296a47bf73990135c7d07ce741b9623b", size = 444782, upload-time = "2025-06-14T08:33:14.905Z" }, +] + +[[package]] +name = "pydantic-core" +version = "2.33.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ad/88/5f2260bdfae97aabf98f1778d43f69574390ad787afb646292a638c923d4/pydantic_core-2.33.2.tar.gz", hash = "sha256:7cb8bc3605c29176e1b105350d2e6474142d7c1bd1d9327c4a9bdb46bf827acc", size = 435195, upload-time = "2025-04-23T18:33:52.104Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/92/b31726561b5dae176c2d2c2dc43a9c5bfba5d32f96f8b4c0a600dd492447/pydantic_core-2.33.2-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:2b3d326aaef0c0399d9afffeb6367d5e26ddc24d351dbc9c636840ac355dc5d8", size = 2028817, upload-time = "2025-04-23T18:30:43.919Z" }, + { url = "https://files.pythonhosted.org/packages/a3/44/3f0b95fafdaca04a483c4e685fe437c6891001bf3ce8b2fded82b9ea3aa1/pydantic_core-2.33.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0e5b2671f05ba48b94cb90ce55d8bdcaaedb8ba00cc5359f6810fc918713983d", size = 1861357, upload-time = "2025-04-23T18:30:46.372Z" }, + { url = "https://files.pythonhosted.org/packages/30/97/e8f13b55766234caae05372826e8e4b3b96e7b248be3157f53237682e43c/pydantic_core-2.33.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0069c9acc3f3981b9ff4cdfaf088e98d83440a4c7ea1bc07460af3d4dc22e72d", size = 1898011, upload-time = "2025-04-23T18:30:47.591Z" }, + { url = "https://files.pythonhosted.org/packages/9b/a3/99c48cf7bafc991cc3ee66fd544c0aae8dc907b752f1dad2d79b1b5a471f/pydantic_core-2.33.2-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d53b22f2032c42eaaf025f7c40c2e3b94568ae077a606f006d206a463bc69572", size = 1982730, upload-time = "2025-04-23T18:30:49.328Z" }, + { url = "https://files.pythonhosted.org/packages/de/8e/a5b882ec4307010a840fb8b58bd9bf65d1840c92eae7534c7441709bf54b/pydantic_core-2.33.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0405262705a123b7ce9f0b92f123334d67b70fd1f20a9372b907ce1080c7ba02", size = 2136178, upload-time = "2025-04-23T18:30:50.907Z" }, + { url = "https://files.pythonhosted.org/packages/e4/bb/71e35fc3ed05af6834e890edb75968e2802fe98778971ab5cba20a162315/pydantic_core-2.33.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4b25d91e288e2c4e0662b8038a28c6a07eaac3e196cfc4ff69de4ea3db992a1b", size = 2736462, upload-time = "2025-04-23T18:30:52.083Z" }, + { url = "https://files.pythonhosted.org/packages/31/0d/c8f7593e6bc7066289bbc366f2235701dcbebcd1ff0ef8e64f6f239fb47d/pydantic_core-2.33.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6bdfe4b3789761f3bcb4b1ddf33355a71079858958e3a552f16d5af19768fef2", size = 2005652, upload-time = "2025-04-23T18:30:53.389Z" }, + { url = "https://files.pythonhosted.org/packages/d2/7a/996d8bd75f3eda405e3dd219ff5ff0a283cd8e34add39d8ef9157e722867/pydantic_core-2.33.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:efec8db3266b76ef9607c2c4c419bdb06bf335ae433b80816089ea7585816f6a", size = 2113306, upload-time = "2025-04-23T18:30:54.661Z" }, + { url = "https://files.pythonhosted.org/packages/ff/84/daf2a6fb2db40ffda6578a7e8c5a6e9c8affb251a05c233ae37098118788/pydantic_core-2.33.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:031c57d67ca86902726e0fae2214ce6770bbe2f710dc33063187a68744a5ecac", size = 2073720, upload-time = "2025-04-23T18:30:56.11Z" }, + { url = "https://files.pythonhosted.org/packages/77/fb/2258da019f4825128445ae79456a5499c032b55849dbd5bed78c95ccf163/pydantic_core-2.33.2-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:f8de619080e944347f5f20de29a975c2d815d9ddd8be9b9b7268e2e3ef68605a", size = 2244915, upload-time = "2025-04-23T18:30:57.501Z" }, + { url = "https://files.pythonhosted.org/packages/d8/7a/925ff73756031289468326e355b6fa8316960d0d65f8b5d6b3a3e7866de7/pydantic_core-2.33.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:73662edf539e72a9440129f231ed3757faab89630d291b784ca99237fb94db2b", size = 2241884, upload-time = "2025-04-23T18:30:58.867Z" }, + { url = "https://files.pythonhosted.org/packages/0b/b0/249ee6d2646f1cdadcb813805fe76265745c4010cf20a8eba7b0e639d9b2/pydantic_core-2.33.2-cp310-cp310-win32.whl", hash = "sha256:0a39979dcbb70998b0e505fb1556a1d550a0781463ce84ebf915ba293ccb7e22", size = 1910496, upload-time = "2025-04-23T18:31:00.078Z" }, + { url = "https://files.pythonhosted.org/packages/66/ff/172ba8f12a42d4b552917aa65d1f2328990d3ccfc01d5b7c943ec084299f/pydantic_core-2.33.2-cp310-cp310-win_amd64.whl", hash = "sha256:b0379a2b24882fef529ec3b4987cb5d003b9cda32256024e6fe1586ac45fc640", size = 1955019, upload-time = "2025-04-23T18:31:01.335Z" }, + { url = "https://files.pythonhosted.org/packages/30/68/373d55e58b7e83ce371691f6eaa7175e3a24b956c44628eb25d7da007917/pydantic_core-2.33.2-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5c4aa4e82353f65e548c476b37e64189783aa5384903bfea4f41580f255fddfa", size = 2023982, upload-time = "2025-04-23T18:32:53.14Z" }, + { url = "https://files.pythonhosted.org/packages/a4/16/145f54ac08c96a63d8ed6442f9dec17b2773d19920b627b18d4f10a061ea/pydantic_core-2.33.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:d946c8bf0d5c24bf4fe333af284c59a19358aa3ec18cb3dc4370080da1e8ad29", size = 1858412, upload-time = "2025-04-23T18:32:55.52Z" }, + { url = "https://files.pythonhosted.org/packages/41/b1/c6dc6c3e2de4516c0bb2c46f6a373b91b5660312342a0cf5826e38ad82fa/pydantic_core-2.33.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:87b31b6846e361ef83fedb187bb5b4372d0da3f7e28d85415efa92d6125d6e6d", size = 1892749, upload-time = "2025-04-23T18:32:57.546Z" }, + { url = "https://files.pythonhosted.org/packages/12/73/8cd57e20afba760b21b742106f9dbdfa6697f1570b189c7457a1af4cd8a0/pydantic_core-2.33.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa9d91b338f2df0508606f7009fde642391425189bba6d8c653afd80fd6bb64e", size = 2067527, upload-time = "2025-04-23T18:32:59.771Z" }, + { url = "https://files.pythonhosted.org/packages/e3/d5/0bb5d988cc019b3cba4a78f2d4b3854427fc47ee8ec8e9eaabf787da239c/pydantic_core-2.33.2-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2058a32994f1fde4ca0480ab9d1e75a0e8c87c22b53a3ae66554f9af78f2fe8c", size = 2108225, upload-time = "2025-04-23T18:33:04.51Z" }, + { url = "https://files.pythonhosted.org/packages/f1/c5/00c02d1571913d496aabf146106ad8239dc132485ee22efe08085084ff7c/pydantic_core-2.33.2-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:0e03262ab796d986f978f79c943fc5f620381be7287148b8010b4097f79a39ec", size = 2069490, upload-time = "2025-04-23T18:33:06.391Z" }, + { url = "https://files.pythonhosted.org/packages/22/a8/dccc38768274d3ed3a59b5d06f59ccb845778687652daa71df0cab4040d7/pydantic_core-2.33.2-pp310-pypy310_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:1a8695a8d00c73e50bff9dfda4d540b7dee29ff9b8053e38380426a85ef10052", size = 2237525, upload-time = "2025-04-23T18:33:08.44Z" }, + { url = "https://files.pythonhosted.org/packages/d4/e7/4f98c0b125dda7cf7ccd14ba936218397b44f50a56dd8c16a3091df116c3/pydantic_core-2.33.2-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:fa754d1850735a0b0e03bcffd9d4b4343eb417e47196e4485d9cca326073a42c", size = 2238446, upload-time = "2025-04-23T18:33:10.313Z" }, + { url = "https://files.pythonhosted.org/packages/ce/91/2ec36480fdb0b783cd9ef6795753c1dea13882f2e68e73bce76ae8c21e6a/pydantic_core-2.33.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:a11c8d26a50bfab49002947d3d237abe4d9e4b5bdc8846a63537b6488e197808", size = 2066678, upload-time = "2025-04-23T18:33:12.224Z" }, +] + +[[package]] +name = "pyflakes" +version = "3.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/45/dc/fd034dc20b4b264b3d015808458391acbf9df40b1e54750ef175d39180b1/pyflakes-3.4.0.tar.gz", hash = "sha256:b24f96fafb7d2ab0ec5075b7350b3d2d2218eab42003821c06344973d3ea2f58", size = 64669, upload-time = "2025-06-20T18:45:27.834Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/2f/81d580a0fb83baeb066698975cb14a618bdbed7720678566f1b046a95fe8/pyflakes-3.4.0-py2.py3-none-any.whl", hash = "sha256:f742a7dbd0d9cb9ea41e9a24a918996e8170c799fa528688d40dd582c8265f4f", size = 63551, upload-time = "2025-06-20T18:45:26.937Z" }, +] + +[[package]] +name = "pygments" +version = "2.19.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631, upload-time = "2025-06-21T13:39:12.283Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, +] + +[[package]] +name = "pytest" +version = "8.4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "exceptiongroup" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, + { name = "tomli" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/08/ba/45911d754e8eba3d5a841a5ce61a65a685ff1798421ac054f85aa8747dfb/pytest-8.4.1.tar.gz", hash = "sha256:7c67fd69174877359ed9371ec3af8a3d2b04741818c51e5e99cc1742251fa93c", size = 1517714, upload-time = "2025-06-18T05:48:06.109Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/29/16/c8a903f4c4dffe7a12843191437d7cd8e32751d5de349d45d3fe69544e87/pytest-8.4.1-py3-none-any.whl", hash = "sha256:539c70ba6fcead8e78eebbf1115e8b589e7565830d7d006a8723f19ac8a0afb7", size = 365474, upload-time = "2025-06-18T05:48:03.955Z" }, +] + +[[package]] +name = "pytest-cov" +version = "6.2.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "coverage", extra = ["toml"] }, + { name = "pluggy" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/18/99/668cade231f434aaa59bbfbf49469068d2ddd945000621d3d165d2e7dd7b/pytest_cov-6.2.1.tar.gz", hash = "sha256:25cc6cc0a5358204b8108ecedc51a9b57b34cc6b8c967cc2c01a4e00d8a67da2", size = 69432, upload-time = "2025-06-12T10:47:47.684Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bc/16/4ea354101abb1287856baa4af2732be351c7bee728065aed451b678153fd/pytest_cov-6.2.1-py3-none-any.whl", hash = "sha256:f5bc4c23f42f1cdd23c70b1dab1bbaef4fc505ba950d53e0081d0730dd7e86d5", size = 24644, upload-time = "2025-06-12T10:47:45.932Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/54/ed/79a089b6be93607fa5cdaedf301d7dfb23af5f25c398d5ead2525b063e17/pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e", size = 130631, upload-time = "2024-08-06T20:33:50.674Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9b/95/a3fac87cb7158e231b5a6012e438c647e1a87f09f8e0d123acec8ab8bf71/PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086", size = 184199, upload-time = "2024-08-06T20:31:40.178Z" }, + { url = "https://files.pythonhosted.org/packages/c7/7a/68bd47624dab8fd4afbfd3c48e3b79efe09098ae941de5b58abcbadff5cb/PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf", size = 171758, upload-time = "2024-08-06T20:31:42.173Z" }, + { url = "https://files.pythonhosted.org/packages/49/ee/14c54df452143b9ee9f0f29074d7ca5516a36edb0b4cc40c3f280131656f/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237", size = 718463, upload-time = "2024-08-06T20:31:44.263Z" }, + { url = "https://files.pythonhosted.org/packages/4d/61/de363a97476e766574650d742205be468921a7b532aa2499fcd886b62530/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b", size = 719280, upload-time = "2024-08-06T20:31:50.199Z" }, + { url = "https://files.pythonhosted.org/packages/6b/4e/1523cb902fd98355e2e9ea5e5eb237cbc5f3ad5f3075fa65087aa0ecb669/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed", size = 751239, upload-time = "2024-08-06T20:31:52.292Z" }, + { url = "https://files.pythonhosted.org/packages/b7/33/5504b3a9a4464893c32f118a9cc045190a91637b119a9c881da1cf6b7a72/PyYAML-6.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180", size = 695802, upload-time = "2024-08-06T20:31:53.836Z" }, + { url = "https://files.pythonhosted.org/packages/5c/20/8347dcabd41ef3a3cdc4f7b7a2aff3d06598c8779faa189cdbf878b626a4/PyYAML-6.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68", size = 720527, upload-time = "2024-08-06T20:31:55.565Z" }, + { url = "https://files.pythonhosted.org/packages/be/aa/5afe99233fb360d0ff37377145a949ae258aaab831bde4792b32650a4378/PyYAML-6.0.2-cp310-cp310-win32.whl", hash = "sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99", size = 144052, upload-time = "2024-08-06T20:31:56.914Z" }, + { url = "https://files.pythonhosted.org/packages/b5/84/0fa4b06f6d6c958d207620fc60005e241ecedceee58931bb20138e1e5776/PyYAML-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e", size = 161774, upload-time = "2024-08-06T20:31:58.304Z" }, +] + +[[package]] +name = "requests" +version = "2.32.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e1/0a/929373653770d8a0d7ea76c37de6e41f11eb07559b103b1c02cafb3f7cf8/requests-2.32.4.tar.gz", hash = "sha256:27d0316682c8a29834d3264820024b62a36942083d52caf2f14c0591336d3422", size = 135258, upload-time = "2025-06-09T16:43:07.34Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7c/e4/56027c4a6b4ae70ca9de302488c5ca95ad4a39e190093d6c1a8ace08341b/requests-2.32.4-py3-none-any.whl", hash = "sha256:27babd3cda2a6d50b30443204ee89830707d396671944c998b5975b031ac2b2c", size = 64847, upload-time = "2025-06-09T16:43:05.728Z" }, +] + +[[package]] +name = "rich" +version = "14.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fe/75/af448d8e52bf1d8fa6a9d089ca6c07ff4453d86c65c145d0a300bb073b9b/rich-14.1.0.tar.gz", hash = "sha256:e497a48b844b0320d45007cdebfeaeed8db2a4f4bcf49f15e455cfc4af11eaa8", size = 224441, upload-time = "2025-07-25T07:32:58.125Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e3/30/3c4d035596d3cf444529e0b2953ad0466f6049528a879d27534700580395/rich-14.1.0-py3-none-any.whl", hash = "sha256:536f5f1785986d6dbdea3c75205c473f970777b4a0d6c6dd1b696aa05a3fa04f", size = 243368, upload-time = "2025-07-25T07:32:56.73Z" }, +] + +[[package]] +name = "tomli" +version = "2.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/18/87/302344fed471e44a87289cf4967697d07e532f2421fdaf868a303cbae4ff/tomli-2.2.1.tar.gz", hash = "sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff", size = 17175, upload-time = "2024-11-27T22:38:36.873Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6e/c2/61d3e0f47e2b74ef40a68b9e6ad5984f6241a942f7cd3bbfbdbd03861ea9/tomli-2.2.1-py3-none-any.whl", hash = "sha256:cb55c73c5f4408779d0cf3eef9f762b9c9f147a77de7b258bef0a5628adc85cc", size = 14257, upload-time = "2024-11-27T22:38:35.385Z" }, +] + +[[package]] +name = "tqdm" +version = "4.67.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a8/4b/29b4ef32e036bb34e4ab51796dd745cdba7ed47ad142a9f4a1eb8e0c744d/tqdm-4.67.1.tar.gz", hash = "sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2", size = 169737, upload-time = "2024-11-24T20:12:22.481Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/30/dc54f88dd4a2b5dc8a0279bdd7270e735851848b762aeb1c1184ed1f6b14/tqdm-4.67.1-py3-none-any.whl", hash = "sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2", size = 78540, upload-time = "2024-11-24T20:12:19.698Z" }, +] + +[[package]] +name = "types-pyyaml" +version = "6.0.12.20250516" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4e/22/59e2aeb48ceeee1f7cd4537db9568df80d62bdb44a7f9e743502ea8aab9c/types_pyyaml-6.0.12.20250516.tar.gz", hash = "sha256:9f21a70216fc0fa1b216a8176db5f9e0af6eb35d2f2932acb87689d03a5bf6ba", size = 17378, upload-time = "2025-05-16T03:08:04.897Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/99/5f/e0af6f7f6a260d9af67e1db4f54d732abad514252a7a378a6c4d17dd1036/types_pyyaml-6.0.12.20250516-py3-none-any.whl", hash = "sha256:8478208feaeb53a34cb5d970c56a7cd76b72659442e733e268a94dc72b2d0530", size = 20312, upload-time = "2025-05-16T03:08:04.019Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.14.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/98/5a/da40306b885cc8c09109dc2e1abd358d5684b1425678151cdaed4731c822/typing_extensions-4.14.1.tar.gz", hash = "sha256:38b39f4aeeab64884ce9f74c94263ef78f3c22467c8724005483154c26648d36", size = 107673, upload-time = "2025-07-04T13:28:34.16Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b5/00/d631e67a838026495268c2f6884f3711a15a9a2a96cd244fdaea53b823fb/typing_extensions-4.14.1-py3-none-any.whl", hash = "sha256:d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76", size = 43906, upload-time = "2025-07-04T13:28:32.743Z" }, +] + +[[package]] +name = "typing-inspection" +version = "0.4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f8/b1/0c11f5058406b3af7609f121aaa6b609744687f1d158b3c3a5bf4cc94238/typing_inspection-0.4.1.tar.gz", hash = "sha256:6ae134cc0203c33377d43188d4064e9b357dba58cff3185f22924610e70a9d28", size = 75726, upload-time = "2025-05-21T18:55:23.885Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/17/69/cd203477f944c353c31bade965f880aa1061fd6bf05ded0726ca845b6ff7/typing_inspection-0.4.1-py3-none-any.whl", hash = "sha256:389055682238f53b04f7badcb49b989835495a96700ced5dab2d8feae4b26f51", size = 14552, upload-time = "2025-05-21T18:55:22.152Z" }, +] + +[[package]] +name = "urllib3" +version = "2.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/15/22/9ee70a2574a4f4599c47dd506532914ce044817c7752a79b6a51286319bc/urllib3-2.5.0.tar.gz", hash = "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760", size = 393185, upload-time = "2025-06-18T14:07:41.644Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a7/c2/fe1e52489ae3122415c51f387e221dd0773709bad6c6cdaa599e8a2c5185/urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc", size = 129795, upload-time = "2025-06-18T14:07:40.39Z" }, +]