litellm/.github/workflows/test-litellm-matrix.yml at main · BerriAI/litellm · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
name: LiteLLM Unit Tests (Matrix)

on:
  pull_request:
    branches: [main]

# Cancel in-progress runs for the same PR
concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  test:
    runs-on: ubuntu-latest
    timeout-minutes: 20 # Increased from 15 to 20
    strategy:
      fail-fast: false
      matrix:
        test-group:
          # tests/test_litellm split by subdirectory (~560 files total)
          # Vertex AI tests separated for better isolation (prevent auth/env pollution)
          - name: "llms-vertex"
            path: "tests/test_litellm/llms/vertex_ai"
            workers: 1
            reruns: 2
          - name: "llms-other"
            path: "tests/test_litellm/llms --ignore=tests/test_litellm/llms/vertex_ai"
            workers: 2
            reruns: 2
          # tests/test_litellm/proxy split by subdirectory (~180 files total)
          - name: "proxy-guardrails"
            path: "tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/management_helpers"
            workers: 2
            reruns: 2
          - name: "proxy-core"
            path: "tests/test_litellm/proxy/auth tests/test_litellm/proxy/client tests/test_litellm/proxy/db tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine"
            workers: 2
            reruns: 2
          - name: "proxy-misc"
            path: "tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/agent_endpoints tests/test_litellm/proxy/anthropic_endpoints tests/test_litellm/proxy/common_utils tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/experimental tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/middleware tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/pass_through_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/response_api_endpoints tests/test_litellm/proxy/spend_tracking tests/test_litellm/proxy/ui_crud_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/test_*.py"
            workers: 2
            reruns: 2
          - name: "integrations"
            path: "tests/test_litellm/integrations"
            workers: 2
            reruns: 3 # Integration tests tend to be flakier
          - name: "core-utils"
            path: "tests/test_litellm/litellm_core_utils"
            workers: 2
            reruns: 1
          - name: "other-1"
            # responses (5942) + caching (1723) + types (819) ≈ 8.5k lines
            path: "tests/test_litellm/responses tests/test_litellm/caching tests/test_litellm/types"
            workers: 2
            reruns: 2
          - name: "other-2"
            # enterprise (3062) + google_genai (2511) + router_utils (1982) ≈ 7.6k lines
            path: "tests/test_litellm/enterprise tests/test_litellm/google_genai tests/test_litellm/router_utils"
            workers: 2
            reruns: 2
          - name: "other-3"
            # remaining dirs ≈ 8.0k lines
            path: "tests/test_litellm/router_strategy tests/test_litellm/secret_managers tests/test_litellm/a2a_protocol tests/test_litellm/anthropic_interface tests/test_litellm/completion_extras tests/test_litellm/containers tests/test_litellm/experimental_mcp_client tests/test_litellm/images tests/test_litellm/interactions tests/test_litellm/passthrough tests/test_litellm/vector_stores"
            workers: 2
            reruns: 2
          - name: "root"
            path: "tests/test_litellm/test_*.py"
            workers: 2
            reruns: 2
          # tests/proxy_unit_tests split alphabetically (~48 files total)
          - name: "proxy-unit-a1"
            # test_[a-j]*.py: jwt (1564) + auth_checks (978) + google_gemini (478) + e2e_pod_lock (437) + rest
            path: "tests/proxy_unit_tests/test_[a-j]*.py"
            workers: 2
            reruns: 1
          - name: "proxy-unit-a2"
            # test_[k-o]*.py: key_generate_prisma (4346) + key_generate_dynamodb + models_fallback
            path: "tests/proxy_unit_tests/test_[k-o]*.py"
            workers: 2
            reruns: 1
          - name: "proxy-unit-b1"
            # lighter config/utility proxy tests (prisma, project, prompt, proxy_[c-r]*)
            path: "tests/proxy_unit_tests/test_prisma*.py tests/proxy_unit_tests/test_project*.py tests/proxy_unit_tests/test_prompt*.py tests/proxy_unit_tests/test_proxy_[c-r]*.py"
            workers: 2
            reruns: 1
          - name: "proxy-unit-b2"
            # proxy_server.py alone (2750 lines) - isolated to avoid blocking smaller tests
            path: "tests/proxy_unit_tests/test_proxy_server.py"
            workers: 2
            reruns: 1
          - name: "proxy-unit-b3"
            # proxy_server_* (618) + proxy_setting_guardrails (71) - smaller server-related tests
            path: "tests/proxy_unit_tests/test_proxy_server_*.py tests/proxy_unit_tests/test_proxy_setting_guardrails.py"
            workers: 2
            reruns: 1
          - name: "proxy-unit-b4"
            # proxy_utils.py alone (2339 lines) - isolated to avoid blocking token counter
            path: "tests/proxy_unit_tests/test_proxy_utils.py"
            workers: 2
            reruns: 1
          - name: "proxy-unit-b5"
            # proxy_token_counter (1279) - runs independently from utils
            path: "tests/proxy_unit_tests/test_proxy_token_counter.py"
            workers: 2
            reruns: 1
          - name: "proxy-unit-b6"
            # test_[r-t]*.py: response_polling (1399) + search_api_logging (202) + server_root (64) + skills_db (261) + realtime_cache (62)
            path: "tests/proxy_unit_tests/test_[r-t]*.py"
            workers: 2
            reruns: 1
          - name: "proxy-unit-b7"
            # test_[u-z]*.py: user_api_key_auth (1136) + zero_cost (590) + update_spend (305) + unit_test_* (206) + ui_path (157)
            path: "tests/proxy_unit_tests/test_[u-z]*.py"
            workers: 2
            reruns: 1

    name: test (${{ matrix.test-group.name }})

    steps:
      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0

      - name: Set up Python
        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
        with:
          python-version: "3.12"

      - name: Install Poetry
        run: pip install 'poetry==2.3.2'

      - name: Cache Poetry dependencies
        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.0.0
        with:
          path: |
            ~/.cache/pypoetry
            ~/.cache/pip
            .venv
          key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }}
          restore-keys: |
            ${{ runner.os }}-poetry-

      - name: Install dependencies
        run: |
          poetry config virtualenvs.in-project true
          poetry install --with dev,proxy-dev --extras "proxy semantic-router"
          # pytest-rerunfailures and pytest-xdist are in pyproject.toml dev dependencies
          poetry run pip install google-genai==1.22.0 \
            google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0

      - name: Setup litellm-enterprise
        run: |
          poetry run pip install --force-reinstall --no-deps -e enterprise/

      - name: Generate Prisma client
        run: |
          poetry run prisma generate --schema litellm/proxy/schema.prisma

      - name: Run tests - ${{ matrix.test-group.name }}
        run: |
          poetry run pytest ${{ matrix.test-group.path }} \
            --tb=short -vv \
            --maxfail=10 \
            -n ${{ matrix.test-group.workers }} \
            --reruns ${{ matrix.test-group.reruns }} \
            --reruns-delay 1 \
            --dist=loadscope \
            --durations=20