more dark theme #302
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Parallel Tests | |
| on: | |
| workflow_dispatch: | |
| pull_request: | |
| branches: [dev, main, master] | |
| push: | |
| branches: [dev, main, master] | |
| env: | |
| PYTHONIOENCODING: utf-8 | |
| PYTHONLEGACYWINDOWSSTDIO: utf-8 | |
| USE_COLOR: False | |
| UV_NO_SOURCES: "1" | |
| jobs: | |
| prek: | |
| name: Pre-commit checks | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Checkout repository | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| git init . | |
| git remote add origin "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" | |
| git fetch --depth=1 origin "${GITHUB_SHA}" | |
| git checkout --detach FETCH_HEAD | |
| - name: Clone abx-pkg | |
| run: git clone --depth=1 https://github.com/ArchiveBox/abx-pkg.git "${{ runner.temp }}/abx-pkg" | |
| - name: Set up Python 3.13 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.13" | |
| architecture: x64 | |
| - name: Install uv | |
| run: python -m pip install --upgrade uv | |
| - name: Set up Node JS | |
| uses: actions/setup-node@v5 | |
| with: | |
| node-version: 22 | |
| - name: Enable pnpm | |
| run: | | |
| corepack enable | |
| echo "PNPM_HOME=$HOME/.local/share/pnpm" >> $GITHUB_ENV | |
| echo "$HOME/.local/share/pnpm" >> $GITHUB_PATH | |
| - name: Install JS tooling | |
| run: pnpm add -g prettier | |
| - name: Cache uv | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/uv | |
| key: ${{ runner.os }}-3.13-uv-${{ hashFiles('pyproject.toml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-3.13-uv- | |
| - name: Install system packages | |
| run: | | |
| sudo apt-get update | |
| sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \ | |
| git wget ripgrep build-essential python3-dev python3-setuptools \ | |
| libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 \ | |
| python3-minimal gnupg2 curl python3-ldap python3-msgpack \ | |
| python3-mutagen python3-regex python3-pycryptodome procps | |
| - name: Install dependencies with uv | |
| run: | | |
| uv venv | |
| uv sync --dev --all-extras --no-sources | |
| uv pip install --no-deps -e "${{ runner.temp }}/abx-pkg" | |
| uv pip install --no-deps -e ".[dev]" | |
| - name: Run prek | |
| env: | |
| PYTHONPATH: ${{ runner.temp }}/abx-pkg | |
| run: uv run --no-sync --no-sources prek run --all-files | |
| discover-tests: | |
| name: Discover test files | |
| runs-on: ubuntu-22.04 | |
| outputs: | |
| test-files: ${{ steps.set-matrix.outputs.test-files }} | |
| steps: | |
| - name: Checkout repository | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| git init . | |
| git remote add origin "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" | |
| git fetch --depth=1 origin "${GITHUB_SHA}" | |
| git checkout --detach FETCH_HEAD | |
| - name: Discover test files | |
| id: set-matrix | |
| run: | | |
| plugin_tests=$(find abx_plugins/plugins -path "*/tests/test_*.py" -type f | sort) | |
| root_tests="" | |
| if [ -d tests ]; then | |
| root_tests=$(find tests -path "*/test_*.py" -type f | sort) | |
| fi | |
| all_tests=$(printf "%s\n%s\n" "$plugin_tests" "$root_tests" | sed '/^$/d') | |
| json_array="[" | |
| first=true | |
| for test_file in $all_tests; do | |
| if [ "$first" = true ]; then | |
| first=false | |
| else | |
| json_array+="," | |
| fi | |
| test_name=$(basename $test_file .py | sed 's/^test_//') | |
| name="$test_name" | |
| needs_chromium=false | |
| needs_brew=false | |
| if [[ "$test_file" == abx_plugins/plugins/*/tests/* ]]; then | |
| plugin_dir="${test_file%/tests/*}" | |
| if grep -Riq "chrom" "$plugin_dir"; then | |
| needs_chromium=true | |
| fi | |
| elif grep -iq "chrom" "$test_file"; then | |
| needs_chromium=true | |
| fi | |
| if [[ "$test_file" == abx_plugins/plugins/brew/tests/* ]]; then | |
| needs_brew=true | |
| fi | |
| json_array+="{\"path\":\"$test_file\",\"name\":\"$name\",\"needs_chromium\":$needs_chromium,\"needs_brew\":$needs_brew}" | |
| done | |
| json_array+="]" | |
| echo "test-files=$json_array" >> $GITHUB_OUTPUT | |
| echo "Found $(echo $all_tests | wc -w) test files" | |
| echo "$json_array" | jq '.' | |
| run-tests: | |
| name: ${{ matrix.test.name }} | |
| runs-on: ubuntu-22.04 | |
| needs: [prek, discover-tests] | |
| env: | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| ABX_PYTEST_TEE_SUBPROCESS_OUTPUT: "1" | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 20 | |
| matrix: | |
| test: ${{ fromJson(needs.discover-tests.outputs.test-files) }} | |
| python: ["3.13"] | |
| steps: | |
| - name: Checkout repository | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| git init . | |
| git remote add origin "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" | |
| git fetch --depth=1 origin "${GITHUB_SHA}" | |
| git checkout --detach FETCH_HEAD | |
| - name: Clone abx-pkg | |
| run: git clone --depth=1 https://github.com/ArchiveBox/abx-pkg.git "${{ runner.temp }}/abx-pkg" | |
| - name: Set up Python ${{ matrix.python }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python }} | |
| architecture: x64 | |
| - name: Install uv | |
| run: python -m pip install --upgrade uv | |
| - name: Set up Node JS | |
| uses: actions/setup-node@v5 | |
| with: | |
| node-version: 22 | |
| - name: Cache uv | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/uv | |
| key: ${{ runner.os }}-${{ matrix.python }}-uv-${{ hashFiles('pyproject.toml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-${{ matrix.python }}-uv- | |
| - name: Install system packages | |
| run: | | |
| sudo apt-get update | |
| sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \ | |
| git wget ripgrep build-essential python3-dev python3-setuptools \ | |
| libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 \ | |
| python3-minimal gnupg2 curl python3-ldap python3-msgpack \ | |
| python3-mutagen python3-regex python3-pycryptodome procps | |
| - name: Install Homebrew | |
| if: ${{ matrix.test.needs_brew }} | |
| env: | |
| NONINTERACTIVE: "1" | |
| run: | | |
| /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" | |
| eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)" | |
| echo "$HOMEBREW_PREFIX/bin" >> $GITHUB_PATH | |
| echo "$HOMEBREW_PREFIX/sbin" >> $GITHUB_PATH | |
| echo "PATH=$PATH" >> $GITHUB_ENV | |
| echo "HOMEBREW_PREFIX=$HOMEBREW_PREFIX" >> $GITHUB_ENV | |
| echo "HOMEBREW_CELLAR=$HOMEBREW_CELLAR" >> $GITHUB_ENV | |
| echo "HOMEBREW_REPOSITORY=$HOMEBREW_REPOSITORY" >> $GITHUB_ENV | |
| echo "MANPATH=$MANPATH" >> $GITHUB_ENV | |
| echo "INFOPATH=$INFOPATH" >> $GITHUB_ENV | |
| echo "XDG_DATA_DIRS=$XDG_DATA_DIRS" >> $GITHUB_ENV | |
| brew --version | |
| - name: Install dependencies with uv | |
| run: | | |
| uv venv | |
| uv sync --dev --all-extras --no-sources | |
| uv pip install --no-deps -e "${{ runner.temp }}/abx-pkg" | |
| uv pip install --no-deps -e ".[dev]" | |
| - name: Verify Chromium launch | |
| if: ${{ matrix.test.needs_chromium }} | |
| uses: ./.github/actions/verify-chromium-launch | |
| - name: Run test - ${{ matrix.test.name }} | |
| env: | |
| PYTHONPATH: ${{ runner.temp }}/abx-pkg | |
| TWOCAPTCHA_API_KEY: ${{ secrets.TWOCAPTCHA_API_KEY }} | |
| CHROME_SANDBOX: "false" | |
| CHROME_HEADLESS: "True" | |
| run: | | |
| uv run --no-sync --no-sources pytest -xvs -rA "${{ matrix.test.path }}" --basetemp="$RUNNER_TEMP/pytest-out" |