test: run the optional tier's payload cases against vLLM CPU servers #6
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: E2E Multimodal (vLLM CPU) | |
| # Multimodal payload acceptance against real vLLM servers in CPU mode: | |
| # e2e/tests/test_vllm_cpu_multimodal.py, one vision-language and one speech | |
| # server per release in e2e/vllm_multimodal_releases.txt. Nightly plus | |
| # on-demand, and on pull requests that touch the payload builders or this | |
| # tier itself. Two model loads per release put it outside the per-change | |
| # budget, and the oracle (does a real server accept what we send) does not | |
| # need to run on every push. | |
| on: | |
| schedule: | |
| - cron: '17 6 * * *' | |
| workflow_dispatch: | |
| pull_request: | |
| branches: | |
| - main | |
| - 'feature/**' | |
| paths: | |
| - '.github/workflows/e2e_multimodal.yml' | |
| - 'e2e/configs/vllm_cpu_multimodal/**' | |
| - 'e2e/tests/test_vllm_cpu_multimodal.py' | |
| - 'e2e/utils/vllm_server.py' | |
| - 'e2e/vllm_cpu_server.sh' | |
| - 'e2e/vllm_multimodal_releases.txt' | |
| - 'inference_perf/datagen/multimodal_sampling.py' | |
| - 'inference_perf/datagen/synthetic/multimodal_datagen.py' | |
| - 'inference_perf/datagen/synthetic/shared_prefix_datagen.py' | |
| - 'inference_perf/datagen/dataset/visionarena_datagen.py' | |
| - 'inference_perf/apis/chat.py' | |
| jobs: | |
| multimodal-e2e: | |
| # ARM, for the same reason the e2e-tests job runs there: the x86 | |
| # ubuntu-latest pool hands out several CPU models, and on the AVX-512 | |
| # masked EPYC draws vLLM's CPU backend cannot build its oneDNN matmul | |
| # primitives and falls back to plain torch. That is survivable for the | |
| # text-only oracle and fatal here: on those draws this slice measured | |
| # `init engine` at 623s against 49s on a clean draw, with servers dying | |
| # in encoder profiling or never finishing it, and cases then timing out | |
| # against the ones that did serve. GitHub's ARM pool is one CPU, so the | |
| # same job on it is deterministic. | |
| runs-on: ubuntu-24.04-arm | |
| timeout-minutes: 120 | |
| env: | |
| # Which models play which oracle is decided in the test module | |
| # (VLM_MODEL / AUDIO_MODEL); these must match it, since this workflow | |
| # starts the containers and the tests only connect to them. | |
| VLM_MODEL: OpenGVLab/InternVL3-1B-hf | |
| AUDIO_MODEL: ibm-granite/granite-4.0-1b-speech | |
| # Media on a CPU VLM needs more context than the 2048 the text-only | |
| # oracle serves with; 4096 is both models' max_position_embeddings. | |
| # Matches MAX_MODEL_LEN in the test module. | |
| VLLM_MAX_MODEL_LEN: "4096" | |
| # Every multimodal model needs its own chat template (the text-only | |
| # template would drop the media placeholders). | |
| VLLM_CHAT_TEMPLATE: "" | |
| # These models profile one encoder item of the maximum feature size | |
| # before they serve (a video for the VLM, an audio clip for the speech | |
| # model), which the text-only oracle has no equivalent of. Two model | |
| # loads plus that pass do not fit the 300s the text-only tier serves | |
| # in. 900s is the budget for a slow runner, not the expected startup. | |
| VLLM_HEALTH_TIMEOUT_SEC: "900" | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| # Same contract as the token-oracle job: the release table lives with | |
| # the tests, and an empty table fails here rather than passing a | |
| # zero-iteration loop below. | |
| - name: Load vLLM version table | |
| run: | | |
| versions="$(grep -vE '^[[:space:]]*(#|$)' e2e/vllm_multimodal_releases.txt | xargs)" | |
| [ -n "$versions" ] | |
| echo "VLLM_VERSIONS=$versions" >> "$GITHUB_ENV" | |
| echo "VLLM_HF_CACHE=$HOME/hf-cache" >> "$GITHUB_ENV" | |
| - name: Cache model weights | |
| uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 | |
| with: | |
| path: ~/hf-cache | |
| key: hf-multimodal-${{ env.VLM_MODEL }}-${{ env.AUDIO_MODEL }} | |
| - name: Prefetch vLLM images | |
| run: | | |
| for v in $VLLM_VERSIONS; do | |
| docker pull "docker.io/vllm/vllm-openai-cpu:$v" > /dev/null 2>&1 & | |
| done | |
| - name: Install Nix | |
| uses: cachix/install-nix-action@4eae8bea4afaa8f8ea8aa638ab9a7fead2f3d21e # v31 | |
| with: | |
| github_access_token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Install dependencies | |
| run: | | |
| nix develop -c pdm sync -d | |
| # For every release: start the vision-language server, run the image | |
| # and video cases, stop it; start the speech server, run the audio | |
| # case, stop it. A pass that skips anything fails, because a case that | |
| # skips against the server meant to serve it means the server is not | |
| # what this workflow thinks it is (wrong model, or the modality table | |
| # in utils/vllm_server.py drifted), and a gate whose oracle silently | |
| # skips gates nothing. Every release runs even after one fails. | |
| - name: Run multimodal e2e tests per vLLM version | |
| run: | | |
| set -uo pipefail | |
| failed="" | |
| # run_pass <version> <model> <modalities> <pytest -k expression> | |
| run_pass() { | |
| local v="$1" model="$2" modalities="$3" select="$4" | |
| local tag="${model##*/}-$v" | |
| local out="test_multimodal_e2e-$tag.out" | |
| if VLLM_MODEL="$model" e2e/vllm_cpu_server.sh start "$v"; then | |
| { | |
| echo "### $model on $v resolved to" | |
| echo '```' | |
| curl -sf http://127.0.0.1:8000/version || true | |
| echo | |
| docker inspect --format '{{index .RepoDigests 0}}' \ | |
| "docker.io/vllm/vllm-openai-cpu:$v" || true | |
| echo '```' | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| if E2E_VLLM_BASE_URL=http://127.0.0.1:8000 E2E_VLLM_MODALITIES="$modalities" \ | |
| nix develop -c pdm run test:e2e:multimodal -k "$select" -rs -o log_cli_level=DEBUG \ | |
| |& tee "$out"; then | |
| if grep -qE '[0-9]+ skipped' "$out"; then | |
| echo "::error::$tag: pass selected '$select' but skipped cases (see $out)" | |
| failed="$failed $tag" | |
| fi | |
| else | |
| failed="$failed $tag" | |
| fi | |
| else | |
| failed="$failed $tag" | |
| fi | |
| e2e/vllm_cpu_server.sh logs > "vllm_server-$tag.log" 2>&1 || true | |
| e2e/vllm_cpu_server.sh stop > /dev/null 2>&1 || true | |
| } | |
| for v in $VLLM_VERSIONS; do | |
| run_pass "$v" "$VLM_MODEL" "image,video" "not audio" | |
| run_pass "$v" "$AUDIO_MODEL" "audio" "audio" | |
| done | |
| if [ -n "$failed" ]; then | |
| echo "::error::multimodal slice failed for:$failed" | |
| exit 1 | |
| fi | |
| - name: Upload e2e outputs | |
| if: always() | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 | |
| with: | |
| name: test_e2e_multimodal | |
| path: | | |
| test_multimodal_e2e-*.out | |
| vllm_server-*.log |