test: run the optional tier's payload cases against vLLM CPU servers #2
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: E2E Multimodal (vLLM CPU) | |
| # Multimodal payload acceptance against real vLLM servers in CPU mode: | |
| # e2e/tests/test_vllm_cpu_multimodal.py, one vision-language and one speech | |
| # server per release in e2e/vllm_multimodal_releases.txt. Nightly plus | |
| # on-demand, and on pull requests that touch the payload builders or this | |
| # tier itself. Two model loads per release put it outside the per-change | |
| # budget, and the oracle (does a real server accept what we send) does not | |
| # need to run on every push. | |
| on: | |
| schedule: | |
| - cron: '17 6 * * *' | |
| workflow_dispatch: | |
| pull_request: | |
| branches: | |
| - main | |
| - 'feature/**' | |
| paths: | |
| - '.github/workflows/e2e_multimodal.yml' | |
| - 'e2e/configs/vllm_cpu_multimodal/**' | |
| - 'e2e/tests/test_vllm_cpu_multimodal.py' | |
| - 'e2e/utils/vllm_server.py' | |
| - 'e2e/vllm_cpu_server.sh' | |
| - 'e2e/vllm_multimodal_releases.txt' | |
| - 'inference_perf/datagen/multimodal_sampling.py' | |
| - 'inference_perf/datagen/synthetic/multimodal_datagen.py' | |
| - 'inference_perf/datagen/synthetic/shared_prefix_datagen.py' | |
| - 'inference_perf/datagen/dataset/visionarena_datagen.py' | |
| - 'inference_perf/apis/chat.py' | |
| jobs: | |
| multimodal-e2e: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 120 | |
| env: | |
| # Which models play which oracle is decided in the test module | |
| # (VLM_MODEL / AUDIO_MODEL); these must match it, since this workflow | |
| # starts the containers and the tests only connect to them. | |
| VLM_MODEL: OpenGVLab/InternVL3-1B-hf | |
| AUDIO_MODEL: ibm-granite/granite-4.0-1b-speech | |
| # Media on a CPU VLM needs more context than the 2048 the text-only | |
| # oracle serves with; 4096 is both models' max_position_embeddings. | |
| # Matches MAX_MODEL_LEN in the test module. | |
| VLLM_MAX_MODEL_LEN: "4096" | |
| # Every multimodal model needs its own chat template (the text-only | |
| # template would drop the media placeholders). | |
| VLLM_CHAT_TEMPLATE: "" | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| # Same contract as the token-oracle job: the release table lives with | |
| # the tests, and an empty table fails here rather than passing a | |
| # zero-iteration loop below. | |
| - name: Load vLLM version table | |
| run: | | |
| versions="$(grep -vE '^[[:space:]]*(#|$)' e2e/vllm_multimodal_releases.txt | xargs)" | |
| [ -n "$versions" ] | |
| echo "VLLM_VERSIONS=$versions" >> "$GITHUB_ENV" | |
| echo "VLLM_HF_CACHE=$HOME/hf-cache" >> "$GITHUB_ENV" | |
| - name: Cache model weights | |
| uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 | |
| with: | |
| path: ~/hf-cache | |
| key: hf-multimodal-${{ env.VLM_MODEL }}-${{ env.AUDIO_MODEL }} | |
| - name: Prefetch vLLM images | |
| run: | | |
| for v in $VLLM_VERSIONS; do | |
| docker pull "docker.io/vllm/vllm-openai-cpu:$v" > /dev/null 2>&1 & | |
| done | |
| - name: Install Nix | |
| uses: cachix/install-nix-action@4eae8bea4afaa8f8ea8aa638ab9a7fead2f3d21e # v31 | |
| with: | |
| github_access_token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Install dependencies | |
| run: | | |
| nix develop -c pdm sync -d | |
| # For every release: start the vision-language server, run the image | |
| # and video cases, stop it; start the speech server, run the audio | |
| # case, stop it. A pass that skips anything fails, because a case that | |
| # skips against the server meant to serve it means the server is not | |
| # what this workflow thinks it is (wrong model, or the modality table | |
| # in utils/vllm_server.py drifted), and a gate whose oracle silently | |
| # skips gates nothing. Every release runs even after one fails. | |
| - name: Run multimodal e2e tests per vLLM version | |
| run: | | |
| set -uo pipefail | |
| failed="" | |
| # run_pass <version> <model> <modalities> <pytest -k expression> | |
| run_pass() { | |
| local v="$1" model="$2" modalities="$3" select="$4" | |
| local tag="${model##*/}-$v" | |
| local out="test_multimodal_e2e-$tag.out" | |
| if VLLM_MODEL="$model" e2e/vllm_cpu_server.sh start "$v"; then | |
| { | |
| echo "### $model on $v resolved to" | |
| echo '```' | |
| curl -sf http://127.0.0.1:8000/version || true | |
| echo | |
| docker inspect --format '{{index .RepoDigests 0}}' \ | |
| "docker.io/vllm/vllm-openai-cpu:$v" || true | |
| echo '```' | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| if E2E_VLLM_BASE_URL=http://127.0.0.1:8000 E2E_VLLM_MODALITIES="$modalities" \ | |
| nix develop -c pdm run test:e2e:multimodal -k "$select" -rs -o log_cli_level=DEBUG \ | |
| |& tee "$out"; then | |
| if grep -qE '[0-9]+ skipped' "$out"; then | |
| echo "::error::$tag: pass selected '$select' but skipped cases (see $out)" | |
| failed="$failed $tag" | |
| fi | |
| else | |
| failed="$failed $tag" | |
| fi | |
| else | |
| failed="$failed $tag" | |
| fi | |
| e2e/vllm_cpu_server.sh logs > "vllm_server-$tag.log" 2>&1 || true | |
| e2e/vllm_cpu_server.sh stop > /dev/null 2>&1 || true | |
| } | |
| for v in $VLLM_VERSIONS; do | |
| run_pass "$v" "$VLM_MODEL" "image,video" "not audio" | |
| run_pass "$v" "$AUDIO_MODEL" "audio" "audio" | |
| done | |
| if [ -n "$failed" ]; then | |
| echo "::error::multimodal slice failed for:$failed" | |
| exit 1 | |
| fi | |
| - name: Upload e2e outputs | |
| if: always() | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 | |
| with: | |
| name: test_e2e_multimodal | |
| path: | | |
| test_multimodal_e2e-*.out | |
| vllm_server-*.log |