pyproject.toml

[project]
name = "py-data-juicer"
dynamic = ["version"]
description = "Data Processing for and with Foundation Models."
authors = [
    {name = "SysML Team of Alibaba Tongyi Lab"}
]
readme = "README.md"
license = {text = "Apache-2.0"}
requires-python = ">=3.10,<4"
urls = {repository = "https://github.com/datajuicer/data-juicer"}
classifiers = [
    "License :: OSI Approved :: Apache Software License",
    "Programming Language :: Python :: 3",
    "Operating System :: OS Independent"
]

# Core dependencies
dependencies = [
    "datasets>=4.7.0", # core data loading
    "fsspec>=2026.1.0", # file system operations; >=2026.1.0 for ArrowFile.size compat with pyarrow>=17
    "pandas", # data manipulation
    "numpy>=1.26.4", # numerical operations
    "loguru", # logging
    "tqdm", # progress bars
    "jsonargparse[signatures]", # configuration
    "jsonlines", # JSONL handling
    "zstandard", # compression
    "lz4", # compression
    "multiprocess==0.70.16", # parallel processing
    "dill==0.3.8", # serialization
    "psutil", # system monitoring
    "pydantic>=2.0", # data validation
    "uv",
    "httpx",
    "emoji==2.2.0", # emoji handling
    "tabulate",
    "beautifulsoup4",
    "requests",
    "wget",
    "streamlit",
    "Pillow",
    "mwparserfromhell",
    "regex", # regular expressions
    "tomli",
    "tomli-w",
    "gitpython",
    "dep-logic",
]

[project.optional-dependencies]
# Generic ML & DL
generic = [
    "torch==2.8.0",  # PyTorch
    "transformers==4.57.1",  # Hugging Face Transformers
    "einops",  # Tensor operations
    "accelerate",  # Model acceleration
    "vllm==0.11.0",  # LLM serving
    "uvloop==0.21.0",  # avoid async error before it's fixed in uvloop
    "onnxruntime",  # ONNX runtime
    "cudf-cu12==25.10.0; sys_platform == 'linux'",  # dataframe on GPU (Linux only)
]

# Computer Vision
vision = [
    "av==13.1.0", # video/audio handling
    "opencv-contrib-python",
    "imagededup",  # Image deduplication
    "diffusers>=0.33.0",  # Diffusion models
    "simple-aesthetics-predictor",  # Image aesthetics
    "scenedetect",  # Scene detection
    "ultralytics",  # YOLO models
    "rembg",  # Background removal
    "decord; (sys_platform == 'linux' and platform_machine == 'x86_64') or (sys_platform == 'win32' and platform_machine == 'AMD64')", # video/audio handling (no aarch64/macOS py3.10+ wheel)
    "qwen-vl-utils==0.0.14",  # for Qwen-VL
    "ptlflow==0.4.1",  # optical flow models collection
    "timm==1.0.22",  # avoid importing issue in the latest v1.0.23
    # --- HumanVBench (CVPR 2026) face/video ops ---
    # These ops additionally need ffmpeg/ffprobe binaries on PATH (VideoLLaMA3
    # video loading) and a flash-attn build matching the local torch/CUDA; the
    # matching prebuilt flash-attn wheel should be installed manually since a
    # plain `pip install flash-attn` triggers a slow source build.
    "deepface==0.0.100",  # face age/gender/race demographic op
    "tf-keras==2.21.0",  # required by deepface on TensorFlow>=2.16
    "imageio>=2.37",  # required by VideoLLaMA3 remote code
    "imageio-ffmpeg>=0.6",  # ffmpeg backend for imageio
    "scipy",  # signal/interpolation/wavfile for face-track ASD ops
]

# Natural Language Processing
nlp = [
    "nltk==3.10.3",  # NLP toolkit
    "easyocr==1.7.1",  # OCR
    "fasttext-predict",  # FastText (no numpy dependency, numpy 2.x safe)
    "kenlm",  # Language modeling
    "sentencepiece",  # Tokenization
    "ftfy",  # Text fixing
    "simhash-pybind",  # Text similarity
    "selectolax",  # HTML parsing
    "nlpaug",  # Text augmentation
    "nlpcda",  # Chinese text augmentation
    "tiktoken",  # Token counting
    "opencc==1.1.9",  # Chinese conversion
    "spacy-pkuseg",  # Chinese segmentation
    "rouge",  # Text evaluation
    "pdfplumber",  # For PDF input and processing
    "spacy==3.8.7",
    "python-docx",  # For docx input and processing
]

# Audio Processing
audio = [
    "librosa>=0.10",
    "resampy",
    "samplerate==0.1.0",
    "torchaudio",  # Audio processing with PyTorch
    "torchcodec==0.7; (sys_platform == 'linux' and platform_machine == 'x86_64') or (sys_platform == 'darwin' and platform_machine == 'arm64') or (sys_platform == 'win32' and platform_machine == 'AMD64')", # only platforms with 0.7 wheels
    "soundfile",  # audio handling
    "ffmpeg-python",
    "audiomentations",
    "funasr==1.3.14",  # SenseVoice-based ASR / speech-emotion ops (HumanVBench)
]

# Distributed Computing
distributed = [
    "ray[default]>=2.51.0",  # distributed computing
    "pandas<3",  # Ray Data versions supported by Data-Juicer are not compatible with pandas 3
    "uvloop==0.21.0",  # avoid async error before it's fixed in uvloop
    "pyspark==3.5.5",  # distributed data processing
    "s3fs",  # S3 filesystem support for cloud storage
    "boto3",  # AWS SDK for S3 operations
    "bitarray", # efficient arrays of booleans
    "pylance",  # to support I/O for lance data in ray mode
]

# Development & Tools
dev = [
    "coverage",
    "pre-commit",  # pre-commit hooks
    "sphinx",
    "sphinx-autobuild",
    "sphinx_copybutton",
    "furo",
    "myst_parser",
    "linkify-it-py",
    "recommonmark",
    "wandb>=0.19.11,<0.22",  # >=0.19.11 lifts protobuf cap to <7, compatible with tensorflow 2.21 (protobuf>=6.31); old <=0.19.0 pin (added in #512 to dodge a 0.19.x bug) forced protobuf<6 and clashed with TF
    "fire",
    "click",
    "toml",  # for yapf configuration
    "pytest",  # testing framework
    "pytest-cov",  # coverage reporting
    "build",  # package building
    "black>=25.1.0",
    "flake8-black",
    "docstring-parser>=0.16",
]

# AI Services & APIs
ai_services = [
    "dashscope",  # Alibaba Cloud AI
    "openai",  # OpenAI API
    "litellm>=1.80.0,<2",  # Allows OpenAI 1.x compatibility with label-studio
    "label-studio>=1.23.0",  # Data labeling
]

# tools, including analysis, mcp, ...
tools = [
    "matplotlib",  # For drawing analysis results
    "wordcloud==1.9.4",  # For drawing word clouds
    "mcp[cli]>=1.7.0",  # For mcp service
    "rank-bm25>=0.2.2",  # For OP search
    "fastapi>=0.110",  # For using data-juicer in a service way
]


# All dependencies (default + all optional)
all = [
    "py-data-juicer[generic]",
    "py-data-juicer[vision]",
    "py-data-juicer[nlp]",
    "py-data-juicer[audio]",
    "py-data-juicer[distributed]",
    "py-data-juicer[dev]",
    "py-data-juicer[ai_services]",
    "py-data-juicer[tools]",
]

[project.scripts]
dj-process = "data_juicer.tools.process_data:main"
dj-analyze = "data_juicer.tools.analyze_data:main"
dj-install = "data_juicer.tools.dj_install:main"
dj-mcp = "data_juicer.tools.mcp_server:main"

[build-system]
requires = [
    "hatchling",
    "uv>=0.1.0",
    "Cython>=0.29",
    "pybind11>=2.6",
    "setuptools>=64",
]
build-backend = "hatchling.build"

[tool.hatch.version]
path = "data_juicer/__init__.py"

[tool.hatch.build.targets.wheel]
packages = ["data_juicer", "tools"]
include = ["uv.lock", "pyproject.toml"]

[tool.hatch.build.targets.wheel.hooks.custom]
path = "hatch_build.py"

[tool.hatch.build]
include = ["uv.lock", "pyproject.toml"]

[tool.hatch.build.targets.wheel.shared-data]
"uv.lock" = "uv.lock"
"pyproject.toml" = "pyproject.toml"

[tool.flake8]
per-file-ignores = [
    "*/__init__.py: F401"
]
max-line-length = 120
extend-ignore = [
    "E203",  # whitespace before ':' (black handles this)
    "E501",  # line too long (black handles this)
    "BLK100",  # black would make changes (black handles this)
    "F541",  # f-string is missing placeholders
]

[tool.black]
line-length = 120
target-version = ['py310']

[tool.isort]
profile = "black"

[tool.pytest.ini_options]
testpaths = ["tests"]
python_files = ["test_*.py"]
python_classes = ["*Test", "Test*"]
python_functions = ["test_*"]
addopts = "-v"
filterwarnings = [
    "ignore::DeprecationWarning",
    "ignore::PendingDeprecationWarning",
]

[tool.uv]
override-dependencies = [
    "opencv-python; sys_platform == 'never'",
    "opencv-python-headless; sys_platform == 'never'",
    "opencv-contrib-python-headless; sys_platform == 'never'", # never install other versions of opencv-python
]