pyproject.toml

[build-system]
requires = ["setuptools>=61.0", "wheel"]
build-backend = "setuptools.build_meta"

[project]
name = "iterabledata"
description = "Iterable data processing Python library"
dynamic = ["version", "readme"]
requires-python = ">=3.10"
authors = [{name = "Ivan Begtin"}]
license = "MIT"
license-files = ["LICENSE*"]
classifiers = [
    "Development Status :: 5 - Production/Stable",
    "Programming Language :: Python",
    "Programming Language :: Python :: 3 :: Only",
    "Programming Language :: Python :: 3.10",
    "Programming Language :: Python :: 3.11",
    "Programming Language :: Python :: 3.12",
    "Environment :: Console",
    "Intended Audience :: Developers",
    "Intended Audience :: System Administrators",
    "Topic :: Software Development",
    "Topic :: Text Processing",
]
keywords = ["json", "jsonl", "csv", "bson", "parquet", "orc", "xml", "xls", "xlsx", "dbf", "dataset", "etl", "data-pipelines"]
dependencies = [
    "chardet>=5.0.0",
    "tqdm>=4.64.0",
]

[project.optional-dependencies]
# Format / feature extras (kept optional to minimize base install footprint).
compression = [
    "lz4>=4.0.0",
    "python-snappy>=0.6.0",
    "python-lzo>=1.12.0",
    "brotli>=1.0.0",
    "brotli_file>=1.0.0",
    "zstandard>=0.19.0",
    "py7zr>=0.20.0",
]
parquet = ["pyarrow>=10.0.0"]
orc = ["pyorc>=1.6.0"]
excel = ["xlrd>=2.0.0", "openpyxl>=3.0.0"]
xml = ["lxml>=4.9.0"]
bson = ["pymongo>=4.0.0"]
dbf = ["dbfread>=2.0.0"]
warc = ["warcio>=1.7.0"]
duckdb = ["duckdb>=0.9.0"]
stats = ["pyreadstat>=1.2.0"]
protobuf = ["protobuf>=4.0.0"]
otlp = ["protobuf>=4.0.0", "ijson>=3.2.0"]
ion = ["ion-python>=1.0.0"]
hdf5 = ["h5py>=3.0.0"]
geospatial = ["geojson>=3.0.0", "pyshp>=2.3.0", "fiona>=1.9.0", "mapbox-vector-tile>=1.2.0", "topojson>=1.5.0"]
lidar = ["laspy>=2.5.0"]
mat = ["scipy>=1.10.0", "numpy>=1.24.0", "h5py>=3.0.0"]
geophysical = ["segyio>=1.9.0", "cfgrib>=0.9.10.0", "xarray>=2023.1.0", "obspy>=1.4.0"]
access = ["access-parser>=0.0.6", "pyodbc>=4.0.0"]
fst = ["fst>=0.9.0"]
toml = ["tomli>=2.0.0", "tomli-w>=1.0.0", "toml>=0.10.0"]
msgpack = ["msgpack>=1.0.0"]
yaml = ["pyyaml>=6.0.0"]
pcap = ["dpkt>=1.9.0"]
netcdf = ["netCDF4>=1.6.0"]
cbor = ["cbor2>=5.0.0"]
cdf = ["spacepy>=0.4.0"]  # NASA CDF; may require CDF C library (see https://cdf.gsfc.nasa.gov)
zarr = ["zarr>=2.16.0", "numpy>=1.24.0"]
mvt = ["mapbox-vector-tile>=1.2.0"]
topojson = ["topojson>=1.5.0"]
feed = ["feedparser>=6.0.0"]
dxf = ["ezdxf>=1.0.0"]
html = ["beautifulsoup4>=4.11.0"]
arff = ["liac-arff>=2.5.0"]
json = ["ijson>=3.2.0"]
vortex = ["vortex-data>=0.56.0"]
pydantic = ["pydantic>=2.0.0"]
dataframes = ["pandas>=1.5.0", "polars>=0.19.0", "dask[dataframe]>=2023.1.0"]
cloud = ["fsspec>=2023.1.0", "s3fs>=2023.1.0", "gcsfs>=2023.1.0", "adlfs>=2023.1.0"]
# AI-powered documentation
ai = ["openai>=1.0.0", "requests>=2.28.0"]
anthropic = ["anthropic>=0.40.0"]
google-genai = ["google-genai>=1.0.0"]
# Agent tool integrations
langchain = ["langchain-core>=0.3.0"]
mcp = ["mcp>=1.6.0"]
agents = ["langchain-core>=0.3.0", "mcp>=1.6.0"]
# Database engines (read-only support)
db = [
    "psycopg2-binary>=2.9.0",  # PostgreSQL
    "pymongo>=4.0.0",  # MongoDB
    "elasticsearch>=8.0.0",  # Elasticsearch/OpenSearch
    "pymysql>=1.0.0",  # MySQL/MariaDB
    "pyodbc>=4.0.0",  # Microsoft SQL Server
    "clickhouse-connect>=0.6.0",  # ClickHouse
]
# SQL databases only
db-sql = [
    "psycopg2-binary>=2.9.0",  # PostgreSQL
    "pymysql>=1.0.0",  # MySQL/MariaDB
    "pyodbc>=4.0.0",  # Microsoft SQL Server
    "clickhouse-connect>=0.6.0",  # ClickHouse
]
# NoSQL databases only
db-nosql = [
    "pymongo>=4.0.0",  # MongoDB
    "elasticsearch>=8.0.0",  # Elasticsearch/OpenSearch
]
# Database ingestion (write support - existing)
db-ingest = [
    "psycopg[binary]>=3.0.0",  # PostgreSQL (psycopg3 for ingestion)
    "pymongo>=4.0.0",  # MongoDB
    "mysql-connector-python>=8.0.0",  # MySQL
    "elasticsearch>=8.0.0",  # Elasticsearch
    "duckdb>=0.9.0",  # DuckDB (also in duckdb extra)
]
# RDF formats (TriG, N3, TriX)
rdf = ["rdflib>=6.0.0"]
# Excel Binary
xlsb = ["pyxlsb>=1.0.0"]
# Graph formats (GraphML, GEXF, DOT)
graph = ["networkx>=3.0.0"]
# Alignment formats (BAM, SAM); FASTA/FASTQ are stdlib, no extra
alignment = ["pysam>=0.21.0"]
# Genomic variant formats (VCF/BCF); shares pysam with alignment
bio = ["pysam>=0.21.0"]
# Lakehouse table formats (Delta Lake, Apache Iceberg, Lance, Apache Hudi, DuckLake)
lakehouse = ["deltalake>=0.15.0", "pyiceberg[sql-sqlite]>=0.5.0", "pylance>=0.9.0", "hudi>=0.1.0", "sqlalchemy>=2.0.0", "pyducklake>=1.0.0", "duckdb>=1.4.0"]
# Apache Paimon (Row + Mosaic files + tables); not folded into lakehouse to keep installs opt-in
paimon-row = ["zstandard>=0.19.0"]
paimon-mosaic = ["paimon-mosaic>=0.2.0", "pyarrow>=10.0.0"]
paimon-table = ["pypaimon>=1.4.2", "pyarrow>=10.0.0"]
paimon = ["zstandard>=0.19.0", "paimon-mosaic>=0.2.0", "pyarrow>=10.0.0", "pypaimon>=1.4.2"]
# DuckLake table format (SQL catalog + Parquet data); also included in lakehouse
ducklake = ["pyducklake>=1.0.0", "duckdb>=1.4.0", "pyarrow>=10.0.0"]
# Long-tail format extras (one per format family)
avro = ["avro>=1.11.0"]
npy = ["numpy>=1.24.0"]
ubj = ["py-ubjson>=0.16.0"]
vcf = ["vobject>=0.9.0"]  # vCard contact files
ods = ["odfpy>=1.4.0"]
rdata = ["pyreadr>=0.5.0"]  # R RData/RDS files
capnp = ["pycapnp>=2.0.0"]
thrift = ["thrift>=0.16.0"]
fbs = ["flatbuffers>=23.0.0"]
edn = ["edn-format>=0.7.0"]
hocon = ["pyhocon>=0.3.60"]
der = ["pyasn1>=0.5.0"]  # ASN.1 DER/BER
bencode = ["bencodepy>=0.9.0"]
ics = ["icalendar>=5.0.0"]
ldif = ["ldif3>=3.2.0"]
# Note: SMILE ('smile-json') and FlexBuffers ('flexbuffers') have no
# pip-installable packages; those formats stay ImportError-only for now.


# Convenience group
all = [
    "lz4>=4.0.0",
    "python-snappy>=0.6.0",
    "python-lzo>=1.12.0",
    "brotli>=1.0.0",
    "brotli_file>=1.0.0",
    "zstandard>=0.19.0",
    "py7zr>=0.20.0",
    "pyarrow>=10.0.0",
    "pyorc>=1.6.0",
    "xlrd>=2.0.0",
    "openpyxl>=3.0.0",
    "lxml>=4.9.0",
    "pymongo>=4.0.0",
    "dbfread>=2.0.0",
    "warcio>=1.7.0",
    "duckdb>=0.9.0",
    "pyreadstat>=1.2.0",
    "protobuf>=4.0.0",
    "ion-python>=1.0.0",
    "h5py>=3.0.0",
    "geojson>=3.0.0",
    "pyshp>=2.3.0",
    "fiona>=1.9.0",
    "laspy>=2.5.0",
    "scipy>=1.10.0",
    "segyio>=1.9.0",
    "cfgrib>=0.9.10.0",
    "xarray>=2023.1.0",
    "obspy>=1.4.0",
    "access-parser>=0.0.6",
    "tomli>=2.0.0",
    "tomli-w>=1.0.0",
    "toml>=0.10.0",
    "msgpack>=1.0.0",
    "pyyaml>=6.0.0",
    "dpkt>=1.9.0",
    "netCDF4>=1.6.0",
    "mapbox-vector-tile>=1.2.0",
    "topojson>=1.5.0",
    "feedparser>=6.0.0",
    "ezdxf>=1.0.0",
    "beautifulsoup4>=4.11.0",
    "liac-arff>=2.5.0",
    "ijson>=3.2.0",
    "vortex-data>=0.56.0",
    "pydantic>=2.0.0",
    "pandas>=1.5.0",
    "polars>=0.19.0",
    "dask[dataframe]>=2023.1.0",
    "fsspec>=2023.1.0",
    "s3fs>=2023.1.0",
    "gcsfs>=2023.1.0",
    "adlfs>=2023.1.0",
    # AI and database extras
    "openai>=1.0.0",
    "requests>=2.28.0",
    # Database engines (read-only)
    "psycopg2-binary>=2.9.0",
    "pymongo>=4.0.0",
    "elasticsearch>=8.0.0",
    "pymysql>=1.0.0",
    "pyodbc>=4.0.0",
    "clickhouse-connect>=0.6.0",
    # Database ingestion (write support)
    "psycopg[binary]>=3.0.0",
    "mysql-connector-python>=8.0.0",
    # RDF, XLSB, graph, alignment
    "rdflib>=6.0.0",
    "pyxlsb>=1.0.0",
    "networkx>=3.0.0",
    "pysam>=0.21.0",
    # CBOR and NASA CDF (spacepy may require the CDF C library)
    "cbor2>=5.0.0",
    "spacepy>=0.4.0",
    # Lakehouse table formats
    "deltalake>=0.15.0",
    "pyiceberg[sql-sqlite]>=0.5.0",
    "pylance>=0.9.0",
    "hudi>=0.1.0",
    "sqlalchemy>=2.0.0",
    # Apache Paimon file formats
    "zstandard>=0.19.0",
    "paimon-mosaic>=0.2.0",
    "pypaimon>=1.4.2",
    # DuckLake
    "pyducklake>=1.0.0",
    "duckdb>=1.4.0",
    # Long-tail format extras
    "avro>=1.11.0",
    "numpy>=1.24.0",
    "py-ubjson>=0.16.0",
    "vobject>=0.9.0",
    "odfpy>=1.4.0",
    "pyreadr>=0.5.0",
    "pycapnp>=2.0.0",
    "thrift>=0.16.0",
    "flatbuffers>=23.0.0",
    "edn-format>=0.7.0",
    "pyhocon>=0.3.60",
    "pyasn1>=0.5.0",
    "bencodepy>=0.9.0",
    "icalendar>=5.0.0",
    "ldif3>=3.2.0",
]

dev = [
    "pytest>=7.0.0",
    "pytest-benchmark>=4.0.0",
    "pytest-cov>=4.1.0",
    "memory-profiler>=0.61.0",
    "psutil>=5.9.0",
    "pytest-xdist>=3.3.0",
    "pytest-timeout>=2.2.0",
    "mock>=4.0.0",
    "ruff>=0.1.0",
    "mypy>=1.0.0",
    # Type stubs for optional dependencies
    "types-chardet>=5.0.0",
    "types-openpyxl>=3.0.0",
    "types-pyyaml>=6.0.0",
    "types-requests>=2.31.0",
    "types-python-dateutil>=2.8.0",
    "types-setuptools>=68.0.0",
    # Note: pandas-stubs and pyarrow-stubs are available but not included
    # as they are large and pandas/pyarrow are already in dev dependencies
    # for testing. Add them if needed for stricter type checking.
    "hypothesis>=6.0.0",
    "pandas>=1.5.0", # Often used in tests or examples (pandas has built-in type stubs)
    # Security tools
    "bandit[toml]>=1.7.5",
    "pip-audit>=2.6.1",
    "safety>=2.3.0",
    # Code quality
    "vulture>=2.10",
    "radon>=6.0.0",
    "pydocstyle[toml]>=6.3.0",
    # Coverage
    "coverage[toml]>=7.3.0",
    # Dependency management
    "pip-tools>=7.3.0",
    "pipdeptree>=2.9.0",
    "pip-licenses>=4.3.0",
    # Build tools
    "build>=1.0.0",
    "twine>=4.0.0",
    "check-wheel-contents>=0.5.0",
]

[project.scripts]
iterable-mcp = "iterable.mcp.server:main"

[project.urls]
Homepage = "https://github.com/datenoio/iterabledata/"
Repository = "https://github.com/datenoio/iterabledata/"
Documentation = "https://datenoio.github.io/iterabledata/"
Changelog = "https://github.com/datenoio/iterabledata/blob/main/CHANGELOG.md"

[tool.setuptools.dynamic]
version = {attr = "iterable.__version__"}
readme = {file = ["README.md"], content-type = "text/markdown"}

[tool.setuptools.packages.find]
include = ["iterable", "iterable.*"]
exclude = ["tests", "tests.*"]

[tool.setuptools.package-data]
iterable = ["py.typed"]

[tool.ruff]
line-length = 120
target-version = "py310"

[tool.ruff.lint]
select = ["E", "F", "I", "B", "UP"]
ignore = []

[tool.pytest.ini_options]
addopts = "--doctest-modules --verbose --cov=iterable --cov-report=term-missing --cov-report=html --cov-report=xml -m 'not stress and not slow and not benchmark and not integration and not performance'"
testpaths = ["tests"]
norecursedirs = ["examples", ".git", ".venv", ".hypothesis", "__pycache__", "*.egg", "build", "dist", "node_modules", "docs"]
python_files = ["test_*.py", "*_test.py"]
python_classes = ["Test*"]
python_functions = ["test_*"]
timeout = 300
timeout_method = "thread"
markers = [
    "stress: marks tests as stress tests (creates large files, may take long time)",
    "slow: marks tests as slow running (may take several minutes)",
    "benchmark: marks tests as benchmarks (use --benchmark-only to run)",
    "performance: marks performance regression gate tests (run in a dedicated CI job)",
    "ai: marks tests related to AI/LLM features",
    "integration: marks tests requiring live external services or API keys",
]

[tool.coverage.run]
source = ["iterable"]
omit = ["*/tests/*", "*/test_*.py", "*/__pycache__/*"]
branch = true

[tool.coverage.report]
precision = 2
show_missing = true
skip_covered = false
# Calibrated against a base environment (no optional format extras installed),
# where optional-dependency modules are import-skipped and count as uncovered.
# Full-extras coverage is tracked in CI via codecov.
fail_under = 55
exclude_lines = [
    "pragma: no cover",
    "def __repr__",
    "raise AssertionError",
    "raise NotImplementedError",
    "if __name__ == .__main__.:",
    "if TYPE_CHECKING:",
    "@abstractmethod",
]

[tool.coverage.html]
directory = "htmlcov"

[tool.bandit]
exclude_dirs = ["tests", "build", "dist", ".venv", "__pycache__", "examples"]
skips = ["B101"]  # Skip assert_used warnings in tests

[tool.pydocstyle]
convention = "google"
match = "(?!test_).*\\.py"
match-dir = "(?!build|dist|tests|__pycache__)"

[tool.mypy]
python_version = "3.10"
warn_return_any = true
warn_unused_configs = true
disallow_untyped_defs = false  # Set to true after fixing all errors
disallow_incomplete_defs = false  # Set to true after fixing all errors
check_untyped_defs = true
disallow_untyped_decorators = false  # Set to true after fixing all errors
no_implicit_optional = true
warn_redundant_casts = true
warn_unused_ignores = true
warn_no_return = true
warn_unreachable = true
strict_equality = true
show_error_codes = true
show_error_context = true
pretty = true

# Per-module options
[[tool.mypy.overrides]]
module = [
    "tests.*",
    "examples.*",
    "dev.*",
]
disallow_untyped_defs = false
ignore_errors = false

# Ignore missing imports for optional dependencies
# Note: Type stubs are available for many optional dependencies via types-* packages
# (e.g., types-pyyaml, types-requests). These are installed as dev dependencies.
# For packages without type stubs, we use ignore_missing_imports.
[[tool.mypy.overrides]]
module = [
    "iterable.datatypes.*",
    "iterable.codecs.*",
    "iterable.db.*",
    "iterable.engines.*",
]
ignore_missing_imports = true  # Many formats have optional dependencies without type stubs

# Strict typing for core modules (plan 3.5 — expand as mypy errors are cleared).
[[tool.mypy.overrides]]
module = [
    "iterable.base",
    "iterable.codec_base",
    "iterable.guidance",
    "iterable.exceptions",
    "iterable.types",
    "iterable.helpers.detect",
    "iterable.helpers.open_iterable",
    "iterable.helpers.content_detection",
    "iterable.helpers.format_registry",
    "iterable.async_base",
    "iterable.helpers.utils",
]
disallow_untyped_defs = true

[[tool.mypy.overrides]]
module = "iterable.dataframe_adapters"
disallow_untyped_defs = true
ignore_missing_imports = true

# Packages with type stubs available (no ignore_missing_imports needed):
# - pandas (pandas-stubs package available, but not installed by default)
# - pyarrow (pyarrow-stubs package available, but not installed by default)
# - pyyaml (types-pyyaml)
# - requests (types-requests)
# - chardet (types-chardet)
# - openpyxl (types-openpyxl)
# - pydantic (built-in type stubs)
# - duckdb (built-in type stubs)
# 
# Packages without type stubs (ignore_missing_imports = true):
# - Most other optional dependencies (pyorc, lxml, pymongo, etc.)