fix dependency groups for dev #315
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Parallel Tests | |
| on: | |
| workflow_dispatch: | |
| pull_request: | |
| branches: [dev, main, master] | |
| push: | |
| branches: [dev, main, master] | |
| env: | |
| PYTHONIOENCODING: utf-8 | |
| PYTHONLEGACYWINDOWSSTDIO: utf-8 | |
| USE_COLOR: False | |
| UV_NO_SOURCES: "1" | |
| jobs: | |
| prek: | |
| name: Pre-commit checks | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v5 | |
| with: | |
| fetch-depth: 1 | |
| - name: Clone abx-pkg | |
| run: git clone --depth=1 https://github.com/ArchiveBox/abx-pkg.git "${{ runner.temp }}/abx-pkg" | |
| - name: Set up Python 3.13 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.13" | |
| architecture: x64 | |
| - name: Install uv | |
| run: python -m pip install --upgrade uv | |
| - name: Set up Node JS | |
| uses: actions/setup-node@v5 | |
| with: | |
| node-version: 22 | |
| - name: Enable pnpm | |
| run: | | |
| corepack enable | |
| echo "PNPM_HOME=$HOME/.local/share/pnpm" >> $GITHUB_ENV | |
| echo "$HOME/.local/share/pnpm" >> $GITHUB_PATH | |
| - name: Install JS tooling | |
| run: pnpm add -g prettier | |
| - name: Cache uv | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/uv | |
| key: ${{ runner.os }}-3.13-uv-${{ hashFiles('pyproject.toml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-3.13-uv- | |
| - name: Install system packages | |
| run: | | |
| sudo apt-get update | |
| sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \ | |
| git wget ripgrep build-essential python3-dev python3-setuptools \ | |
| libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 \ | |
| python3-minimal gnupg2 curl python3-ldap python3-msgpack \ | |
| python3-mutagen python3-regex python3-pycryptodome procps | |
| - name: Install dependencies with uv | |
| run: | | |
| uv venv | |
| uv sync --dev --all-extras --no-sources | |
| uv pip install --no-deps -e "${{ runner.temp }}/abx-pkg" | |
| uv pip install --no-deps -e ".[dev]" | |
| - name: Run prek | |
| env: | |
| PYTHONPATH: ${{ runner.temp }}/abx-pkg | |
| run: uv run --no-sync --no-sources prek run --all-files | |
| changes: | |
| name: Detect changed paths | |
| runs-on: ubuntu-22.04 | |
| outputs: | |
| docs-only: ${{ steps.changed-files.outputs.docs-only }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v5 | |
| with: | |
| fetch-depth: 2 | |
| - name: Detect docs-only changes | |
| id: changed-files | |
| run: | | |
| docs_only=false | |
| if git rev-parse --verify HEAD^ >/dev/null 2>&1; then | |
| changed_files=$(git diff --name-only HEAD^ HEAD | sed '/^$/d') | |
| if [ -n "$changed_files" ] && ! printf '%s\n' "$changed_files" | grep -qv '^docs/'; then | |
| docs_only=true | |
| fi | |
| fi | |
| echo "docs-only=$docs_only" >> "$GITHUB_OUTPUT" | |
| echo "docs-only=$docs_only" | |
| discover-tests: | |
| name: Discover test files | |
| runs-on: ubuntu-22.04 | |
| needs: [changes] | |
| if: ${{ needs.changes.outputs.docs-only != 'true' }} | |
| outputs: | |
| test-files: ${{ steps.set-matrix.outputs.test-files }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v5 | |
| with: | |
| fetch-depth: 1 | |
| - name: Discover test files | |
| id: set-matrix | |
| run: | | |
| plugin_tests=$(find abx_plugins/plugins -path "*/tests/test_*.py" -type f | sort) | |
| root_tests="" | |
| if [ -d tests ]; then | |
| root_tests=$(find tests -path "*/test_*.py" -type f | sort) | |
| fi | |
| all_tests=$(printf "%s\n%s\n" "$plugin_tests" "$root_tests" | sed '/^$/d') | |
| json_array="[" | |
| first=true | |
| for test_file in $all_tests; do | |
| if [ "$first" = true ]; then | |
| first=false | |
| else | |
| json_array+="," | |
| fi | |
| test_name=$(basename $test_file .py | sed 's/^test_//') | |
| name="$test_name" | |
| needs_chromium=false | |
| needs_brew=false | |
| if [[ "$test_file" == abx_plugins/plugins/*/tests/* ]]; then | |
| plugin_dir="${test_file%/tests/*}" | |
| if grep -Riq "chrom" "$plugin_dir"; then | |
| needs_chromium=true | |
| fi | |
| elif grep -iq "chrom" "$test_file"; then | |
| needs_chromium=true | |
| fi | |
| if [[ "$test_file" == abx_plugins/plugins/brew/tests/* ]]; then | |
| needs_brew=true | |
| fi | |
| json_array+="{\"path\":\"$test_file\",\"name\":\"$name\",\"needs_chromium\":$needs_chromium,\"needs_brew\":$needs_brew}" | |
| done | |
| json_array+="]" | |
| echo "test-files=$json_array" >> $GITHUB_OUTPUT | |
| echo "Found $(echo $all_tests | wc -w) test files" | |
| echo "$json_array" | jq '.' | |
| run-tests: | |
| name: ${{ matrix.test.name }} | |
| runs-on: ubuntu-22.04 | |
| needs: [prek, changes, discover-tests] | |
| if: ${{ needs.changes.outputs.docs-only != 'true' }} | |
| env: | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| ABX_PYTEST_TEE_SUBPROCESS_OUTPUT: "1" | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 20 | |
| matrix: | |
| test: ${{ fromJson(needs.discover-tests.outputs.test-files) }} | |
| python: ["3.13"] | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v5 | |
| with: | |
| fetch-depth: 1 | |
| - name: Clone abx-pkg | |
| run: git clone --depth=1 https://github.com/ArchiveBox/abx-pkg.git "${{ runner.temp }}/abx-pkg" | |
| - name: Set up Python ${{ matrix.python }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python }} | |
| architecture: x64 | |
| - name: Install uv | |
| run: python -m pip install --upgrade uv | |
| - name: Set up Node JS | |
| uses: actions/setup-node@v5 | |
| with: | |
| node-version: 22 | |
| - name: Cache uv | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/uv | |
| key: ${{ runner.os }}-${{ matrix.python }}-uv-${{ hashFiles('pyproject.toml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-${{ matrix.python }}-uv- | |
| - name: Install system packages | |
| run: | | |
| sudo apt-get update | |
| sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \ | |
| git wget ripgrep build-essential python3-dev python3-setuptools \ | |
| libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 \ | |
| python3-minimal gnupg2 curl python3-ldap python3-msgpack \ | |
| python3-mutagen python3-regex python3-pycryptodome procps | |
| - name: Install Homebrew | |
| if: ${{ matrix.test.needs_brew }} | |
| env: | |
| NONINTERACTIVE: "1" | |
| run: | | |
| /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" | |
| eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)" | |
| echo "$HOMEBREW_PREFIX/bin" >> $GITHUB_PATH | |
| echo "$HOMEBREW_PREFIX/sbin" >> $GITHUB_PATH | |
| echo "PATH=$PATH" >> $GITHUB_ENV | |
| echo "HOMEBREW_PREFIX=$HOMEBREW_PREFIX" >> $GITHUB_ENV | |
| echo "HOMEBREW_CELLAR=$HOMEBREW_CELLAR" >> $GITHUB_ENV | |
| echo "HOMEBREW_REPOSITORY=$HOMEBREW_REPOSITORY" >> $GITHUB_ENV | |
| echo "MANPATH=$MANPATH" >> $GITHUB_ENV | |
| echo "INFOPATH=$INFOPATH" >> $GITHUB_ENV | |
| echo "XDG_DATA_DIRS=$XDG_DATA_DIRS" >> $GITHUB_ENV | |
| brew --version | |
| - name: Install dependencies with uv | |
| run: | | |
| uv venv | |
| uv sync --dev --all-extras --no-sources | |
| uv pip install --no-deps -e "${{ runner.temp }}/abx-pkg" | |
| uv pip install --no-deps -e ".[dev]" | |
| - name: Verify Chromium launch | |
| if: ${{ matrix.test.needs_chromium }} | |
| uses: ./.github/actions/verify-chromium-launch | |
| - name: Run test - ${{ matrix.test.name }} | |
| env: | |
| PYTHONPATH: ${{ runner.temp }}/abx-pkg | |
| TWOCAPTCHA_API_KEY: ${{ secrets.TWOCAPTCHA_API_KEY }} | |
| CHROME_SANDBOX: "false" | |
| CHROME_HEADLESS: "True" | |
| run: | | |
| uv run --no-sync --no-sources pytest -xvs -rA "${{ matrix.test.path }}" --basetemp="$RUNNER_TEMP/pytest-out" |