Skip to content

Commit a469279

Browse files
dolfim-ibmdolfim-ibm
authored
ci: Refactor using uv for dependencies and add package CD (#113)
* refactor for using uv Signed-off-by: Michele Dolfi <[email protected]> * fix deprecated classifier Signed-off-by: Michele Dolfi <[email protected]> * missing uv.lock Signed-off-by: Michele Dolfi <[email protected]> * move xmltodict to deps Signed-off-by: Michele Dolfi <[email protected]> --------- Signed-off-by: Michele Dolfi <[email protected]>
1 parent ee767d1 commit a469279

31 files changed

+5362
-8228
lines changed

‎.github/actions/setup-poetry/action.yml‎

Lines changed: 0 additions & 19 deletions
This file was deleted.

‎.github/codecov.yml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
codecov:
2+
# https://docs.codecov.io/docs/comparing-commits
3+
allow_coverage_offsets: true
4+
coverage:
5+
status:
6+
project:
7+
default:
8+
informational: true
9+
target: auto # auto compares coverage to the previous base commit
10+
if_ci_failed: success
11+
flags:
12+
- docling-eval
13+
comment:
14+
layout: "reach, diff, flags, files"
15+
behavior: default
16+
require_changes: false # if true: only post the comment if coverage changes
17+
branches: # branch names that can post comment
18+
- "main"

‎.github/mergify.yml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
merge_protections:
2+
- name: Enforce conventional commit
3+
description: Make sure that we follow https://www.conventionalcommits.org/en/v1.0.0/
4+
if:
5+
- base = main
6+
success_conditions:
7+
- "title ~=
8+
^(fix|feat|docs|style|refactor|perf|test|build|ci|chore|revert)(?:\\(.+\
9+
\\))?(!)?:"
10+
- name: Require two reviewer for test updates
11+
description: When test data is updated, we require two reviewers
12+
if:
13+
- base = main
14+
- files ~= ^test
15+
success_conditions:
16+
- "#approved-reviews-by >= 2"

‎.github/scripts/release.sh‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
#!/bin/bash
2+
3+
set -e # trigger failure on error - do not remove!
4+
set -x # display command on output
5+
6+
if [ -z "${TARGET_VERSION}" ]; then
7+
>&2 echo "No TARGET_VERSION specified"
8+
exit 1
9+
fi
10+
CHGLOG_FILE="${CHGLOG_FILE:-CHANGELOG.md}"
11+
12+
# update package version
13+
uvx --from=toml-cli toml set --toml-path=pyproject.toml project.version "${TARGET_VERSION}"
14+
uv lock --upgrade-package docling-eval
15+
16+
# collect release notes
17+
REL_NOTES=$(mktemp)
18+
uv run --no-sync semantic-release changelog --unreleased >> "${REL_NOTES}"
19+
20+
# update changelog
21+
TMP_CHGLOG=$(mktemp)
22+
TARGET_TAG_NAME="v${TARGET_VERSION}"
23+
RELEASE_URL="$(gh repo view --json url -q ".url")/releases/tag/${TARGET_TAG_NAME}"
24+
printf "## [${TARGET_TAG_NAME}](${RELEASE_URL}) - $(date -Idate)\n\n" >> "${TMP_CHGLOG}"
25+
cat "${REL_NOTES}" >> "${TMP_CHGLOG}"
26+
if [ -f "${CHGLOG_FILE}" ]; then
27+
printf "\n" | cat - "${CHGLOG_FILE}" >> "${TMP_CHGLOG}"
28+
fi
29+
mv "${TMP_CHGLOG}" "${CHGLOG_FILE}"
30+
31+
# push changes
32+
git config --global user.name 'github-actions[bot]'
33+
git config --global user.email 'github-actions[bot]@users.noreply.github.com'
34+
git add pyproject.toml uv.lock "${CHGLOG_FILE}"
35+
COMMIT_MSG="chore: bump version to ${TARGET_VERSION} [skip ci]"
36+
git commit -m "${COMMIT_MSG}"
37+
git push origin main
38+
39+
# create GitHub release (incl. Git tag)
40+
gh release create "${TARGET_TAG_NAME}" -F "${REL_NOTES}"

‎.github/workflows/cd.yml‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
name: "Run CD"
2+
3+
on:
4+
workflow_dispatch:
5+
6+
jobs:
7+
code-checks:
8+
uses: ./.github/workflows/checks.yml
9+
with:
10+
push_coverage: false
11+
pre-release-check:
12+
runs-on: ubuntu-latest
13+
outputs:
14+
TARGET_TAG_V: ${{ steps.version_check.outputs.TRGT_VERSION }}
15+
steps:
16+
- uses: actions/checkout@v4
17+
with:
18+
fetch-depth: 0 # for fetching tags, required for semantic-release
19+
- name: Install uv and set the python version
20+
uses: astral-sh/setup-uv@v5
21+
with:
22+
enable-cache: true
23+
- name: Install dependencies
24+
run: uv sync --only-dev
25+
- name: Check version of potential release
26+
id: version_check
27+
run: |
28+
TRGT_VERSION=$(uv run --no-sync semantic-release print-version)
29+
echo "TRGT_VERSION=${TRGT_VERSION}" >> "$GITHUB_OUTPUT"
30+
echo "${TRGT_VERSION}"
31+
- name: Check notes of potential release
32+
run: uv run --no-sync semantic-release changelog --unreleased
33+
release:
34+
needs: [code-checks, pre-release-check]
35+
if: needs.pre-release-check.outputs.TARGET_TAG_V != ''
36+
environment: auto-release
37+
runs-on: ubuntu-latest
38+
concurrency: release
39+
steps:
40+
- uses: actions/create-github-app-token@v1
41+
id: app-token
42+
with:
43+
app-id: ${{ vars.CI_APP_ID }}
44+
private-key: ${{ secrets.CI_PRIVATE_KEY }}
45+
- uses: actions/checkout@v4
46+
with:
47+
token: ${{ steps.app-token.outputs.token }}
48+
fetch-depth: 0 # for fetching tags, required for semantic-release
49+
- name: Install uv and set the python version
50+
uses: astral-sh/setup-uv@v5
51+
with:
52+
enable-cache: true
53+
- name: Install dependencies
54+
run: uv sync --only-dev
55+
- name: Run release script
56+
env:
57+
GH_TOKEN: ${{ steps.app-token.outputs.token }}
58+
TARGET_VERSION: ${{ needs.pre-release-check.outputs.TARGET_TAG_V }}
59+
CHGLOG_FILE: CHANGELOG.md
60+
run: ./.github/scripts/release.sh
61+
shell: bash

‎.github/workflows/checks.yml‎

Lines changed: 82 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1,44 +1,96 @@
11
on:
22
workflow_call:
3+
inputs:
4+
push_coverage:
5+
type: boolean
6+
description: "If true, the coverage results are pushed to codecov.io."
7+
default: true
8+
secrets:
9+
CODECOV_TOKEN:
10+
required: false
311

412
env:
5-
RUN_IN_CI: "1"
13+
HF_HUB_DOWNLOAD_TIMEOUT: "60"
14+
HF_HUB_ETAG_TIMEOUT: "60"
615

716
jobs:
817
run-checks:
918
runs-on: ubuntu-latest
1019
strategy:
1120
matrix:
12-
python-version: ['3.10', '3.11', '3.12']
21+
python-version: ['3.10', '3.11', '3.12', '3.13']
1322
steps:
1423
- uses: actions/checkout@v4
15-
- name: Install tesseract
16-
run: sudo apt-get update && sudo apt-get install -y tesseract-ocr tesseract-ocr-eng tesseract-ocr-fra tesseract-ocr-deu tesseract-ocr-spa libleptonica-dev libtesseract-dev pkg-config
17-
- name: Set TESSDATA_PREFIX
18-
run: |
19-
echo "TESSDATA_PREFIX=$(dpkg -L tesseract-ocr-eng | grep tessdata$)" >> "$GITHUB_ENV"
20-
- uses: ./.github/actions/setup-poetry
24+
- name: Cache Hugging Face models
25+
uses: actions/cache@v4
26+
with:
27+
path: ~/.cache/huggingface
28+
key: huggingface-cache-py${{ matrix.python-version }}
29+
- name: Install uv and set the python version
30+
uses: astral-sh/setup-uv@v5
31+
with:
32+
python-version: ${{ matrix.python-version }}
33+
enable-cache: true
34+
- name: pre-commit cache key
35+
run: echo "PY=$(python -VV | sha256sum | cut -d' ' -f1)" >> "$GITHUB_ENV"
36+
- uses: actions/cache@v4
37+
with:
38+
path: ~/.cache/pre-commit
39+
key: pre-commit|${{ env.PY }}|${{ hashFiles('.pre-commit-config.yaml') }}
40+
- name: Install dependencies
41+
run: uv sync --frozen --all-extras
42+
- name: Check style and run tests
43+
run: pre-commit run --all-files
44+
- name: Upload coverage to Codecov
45+
if: inputs.push_coverage
46+
uses: codecov/codecov-action@v5
47+
with:
48+
token: ${{ secrets.CODECOV_TOKEN }}
49+
files: ./coverage.xml
50+
51+
build-package:
52+
runs-on: ubuntu-latest
53+
strategy:
54+
matrix:
55+
python-version: ['3.12']
56+
steps:
57+
- uses: actions/checkout@v4
58+
- name: Install uv and set the python version
59+
uses: astral-sh/setup-uv@v5
60+
with:
61+
python-version: ${{ matrix.python-version }}
62+
enable-cache: true
63+
- name: Install dependencies
64+
run: uv sync --all-extras
65+
- name: Build package
66+
run: uv build
67+
- name: Check content of wheel
68+
run: unzip -l dist/*.whl
69+
- name: Store the distribution packages
70+
uses: actions/upload-artifact@v4
71+
with:
72+
name: python-package-distributions
73+
path: dist/
74+
75+
test-package:
76+
needs:
77+
- build-package
78+
runs-on: ubuntu-latest
79+
strategy:
80+
matrix:
81+
python-version: ['3.12']
82+
steps:
83+
- name: Download all the dists
84+
uses: actions/download-artifact@v4
85+
with:
86+
name: python-package-distributions
87+
path: dist/
88+
- name: Install uv and set the python version
89+
uses: astral-sh/setup-uv@v5
2190
with:
2291
python-version: ${{ matrix.python-version }}
23-
- name: Run styling check
24-
run: poetry run pre-commit run --all-files
25-
- name: Install with poetry
26-
run: poetry install --all-extras
27-
- name: Testing
28-
run: |
29-
poetry run pytest -v tests
30-
- name: Run examples
31-
run: |
32-
# for file in docs/examples/*.py; do
33-
# # Skip batch_convert.py
34-
# if [[ "$(basename "$file")" =~ ^(batch_convert|minimal|export_multimodal|custom_convert|develop_picture_enrichment).py ]]; then
35-
# echo "Skipping $file"
36-
# continue
37-
# fi
38-
#
39-
# echo "Running example $file"
40-
# poetry run python "$file" || exit 1
41-
# done
42-
echo "Skipping examples in CI until this is more time efficient."
43-
- name: Build with poetry
44-
run: poetry build
92+
enable-cache: true
93+
- name: Install package
94+
run: uv pip install dist/*.whl
95+
- name: Run docling-eval
96+
run: docling-eval --help

‎.github/workflows/ci.yml‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,12 @@ on:
66
push:
77
branches:
88
- "**"
9-
- "!main"
109
- "!gh-pages"
1110

12-
env:
13-
# disable keyring (https://github.com/actions/runner-images/issues/6185):
14-
PYTHON_KEYRING_BACKEND: keyring.backends.null.Keyring
15-
1611
jobs:
1712
code-checks:
18-
if: ${{ github.event_name == 'push' || (github.event.pull_request.head.repo.full_name != 'DS4SD/docling-eval' && github.event.pull_request.head.repo.full_name != 'ds4sd/docling-eval') }}
13+
if: ${{ github.event_name == 'push' || (github.event.pull_request.head.repo.full_name != 'docling-project/docling-eval' && github.event.pull_request.head.repo.full_name != 'docling-project/docling-eval') }}
1914
uses: ./.github/workflows/checks.yml
15+
secrets:
16+
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
17+

‎.github/workflows/pypi.yml‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
name: "Build and publish package"
2+
3+
on:
4+
release:
5+
types: [published]
6+
7+
permissions:
8+
contents: read
9+
10+
jobs:
11+
build-and-publish:
12+
runs-on: ubuntu-latest
13+
strategy:
14+
matrix:
15+
python-version: ['3.12']
16+
environment:
17+
name: pypi
18+
url: https://pypi.org/p/docling-eval
19+
permissions:
20+
id-token: write # IMPORTANT: mandatory for trusted publishing
21+
steps:
22+
- uses: actions/checkout@v4
23+
- name: Install uv and set the python version
24+
uses: astral-sh/setup-uv@v5
25+
with:
26+
python-version: ${{ matrix.python-version }}
27+
enable-cache: true
28+
- name: Install dependencies
29+
run: uv sync --all-extras
30+
- name: Build package
31+
run: uv build
32+
- name: Publish distribution 📦 to PyPI
33+
uses: pypa/gh-action-pypi-publish@release/v1
34+
with:
35+
attestations: true

‎.pre-commit-config.yaml‎

Lines changed: 7 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -4,30 +4,23 @@ repos:
44
hooks:
55
- id: black
66
name: Black
7-
entry: poetry run black docling_eval tests docs/examples
7+
entry: uv run --no-sync black docling_eval tests docs/examples
88
pass_filenames: false
99
language: system
1010
files: '\.py$'
1111
- id: isort
1212
name: isort
13-
entry: poetry run isort docling_eval tests docs/examples
13+
entry: uv run --no-sync isort docling_eval tests docs/examples
1414
pass_filenames: false
1515
language: system
1616
files: '\.py$'
17-
# - id: flake8
18-
# name: flake8
19-
# entry: poetry run flake8 docling
20-
# pass_filenames: false
21-
# language: system
22-
# files: '\.py$'
2317
- id: mypy
2418
name: MyPy
25-
entry: poetry run mypy docling_eval tests docs/examples
19+
entry: uv run --no-sync mypy docling_eval tests docs/examples
2620
pass_filenames: false
2721
language: system
2822
files: '\.py$'
29-
- id: poetry
30-
name: Poetry check
31-
entry: poetry check --lock
32-
pass_filenames: false
33-
language: system
23+
- repo: https://github.com/astral-sh/uv-pre-commit
24+
rev: 0.7.8
25+
hooks:
26+
- id: uv-lock

0 commit comments

Comments
 (0)