abx-plugins/.github/workflows/test-parallel.yml at 9808e97b32e8ebaba34979c648073b81cd8602f4 · ArchiveBox/abx-plugins · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
name: Parallel Tests

on:
  workflow_dispatch:
  pull_request:
    branches: [dev, main, master]
  push:
    branches: [dev, main, master]

env:
  PYTHONIOENCODING: utf-8
  PYTHONLEGACYWINDOWSSTDIO: utf-8
  USE_COLOR: False

jobs:
  discover-tests:
    name: Discover test files
    runs-on: ubuntu-22.04
    outputs:
      test-files: ${{ steps.set-matrix.outputs.test-files }}

    steps:
      - name: Checkout repository
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          git init .
          git remote add origin "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git"
          git fetch --depth=1 origin "${GITHUB_SHA}"
          git checkout --detach FETCH_HEAD

      - name: Discover test files
        id: set-matrix
        run: |
          chrome_test_pattern='ensure_chrome_test_prereqs|ensure_chromium_and_puppeteer_installed|require_chrome_runtime|chrome_session\(|CHROME_NAVIGATE_HOOK'
          plugin_tests=$(find abx_plugins/plugins -path "*/tests/test_*.py" -type f | sort)
          root_tests=""
          if [ -d tests ]; then
            root_tests=$(find tests -path "*/test_*.py" -type f | sort)
          fi
          all_tests=$(printf "%s\n%s\n" "$plugin_tests" "$root_tests" | sed '/^$/d')

          json_array="["
          first=true
          for test_file in $all_tests; do
            if [ "$first" = true ]; then
              first=false
            else
              json_array+=","
            fi

            test_name=$(basename $test_file .py | sed 's/^test_//')
            name="$test_name"
            needs_chromium=false
            if rg -q "$chrome_test_pattern" "$test_file"; then
              needs_chromium=true
            fi

            json_array+="{\"path\":\"$test_file\",\"name\":\"$name\",\"needs_chromium\":$needs_chromium}"
          done
          json_array+="]"

          echo "test-files=$json_array" >> $GITHUB_OUTPUT
          echo "Found $(echo $all_tests | wc -w) test files"
          echo "$json_array" | jq '.'

  run-tests:
    name: ${{ matrix.test.name }}
    runs-on: ubuntu-22.04
    needs: discover-tests

    strategy:
      fail-fast: false
      max-parallel: 8
      matrix:
        test: ${{ fromJson(needs.discover-tests.outputs.test-files) }}
        python: ["3.13"]

    steps:
      - name: Checkout repository
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          git init .
          git remote add origin "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git"
          git fetch --depth=1 origin "${GITHUB_SHA}"
          git checkout --detach FETCH_HEAD

      - name: Set up Python ${{ matrix.python }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
          architecture: x64

      - name: Install uv
        run: python -m pip install --upgrade uv

      - name: Set up Node JS
        uses: actions/setup-node@v5
        with:
          node-version: 22

      - name: Cache uv
        uses: actions/cache@v4
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ matrix.python }}-uv-${{ hashFiles('pyproject.toml') }}
          restore-keys: |
            ${{ runner.os }}-${{ matrix.python }}-uv-

      - name: Install system packages
        run: |
          sudo apt-get update
          sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \
            git wget ripgrep build-essential python3-dev python3-setuptools \
            libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 \
            python3-minimal gnupg2 curl python3-ldap python3-msgpack \
            python3-mutagen python3-regex python3-pycryptodome procps

      - name: Install dependencies with uv
        run: |
          uv venv
          uv sync --dev --all-extras
          uv pip install -e ".[dev]"

      - name: Verify Chromium launch
        if: ${{ matrix.test.needs_chromium }}
        uses: ./.github/actions/verify-chromium-launch

      - name: Run test - ${{ matrix.test.name }}
        run: |
          uv run pytest -xvs "${{ matrix.test.path }}" --basetemp="$RUNNER_TEMP/pytest-out"
        env:
          TWOCAPTCHA_API_KEY: ${{ secrets.TWOCAPTCHA_API_KEY }}
          CHROME_SANDBOX: "false"
          CHROME_HEADLESS: "True"