Skip to content

Commit 243b97f

Browse files
Bslabe123Bslabe123
committed
test: catch SGLang and TGI metric-name drift
1 parent 7bfbaed commit 243b97f

6 files changed

Lines changed: 1104 additions & 0 deletions

File tree

‎.github/workflows/metric_name_drift.yml‎

Lines changed: 214 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,214 @@
1+
# Periodic metric-name drift detection for SGLang and TGI (#669).
2+
#
3+
# inference-perf hardcodes the Prometheus metric names it queries per model
4+
# server. When an upstream server renames, removes or re-namespaces one,
5+
# nothing fails: the PromQL matches nothing, the report field comes back
6+
# empty, and a reader cannot tell "the server did not report this" from "we
7+
# asked for a name that no longer exists". #382 caught one such rename
8+
# (sglang:cache_hit_rate to sglang:token_usage) by hand.
9+
#
10+
# This job starts the LATEST release of each server, scrapes /metrics once,
11+
# and rewrites e2e/testdata/server_metric_families/<server>.txt. If the family
12+
# list changed it opens a pull request; if not it exits quietly. The pull
13+
# request then runs the normal checks in
14+
# e2e/tests/test_sglang_tgi_metric_names.py, so a name we still declare but
15+
# the server no longer exposes goes red on one attributable PR that somebody
16+
# owns, rather than on every open PR at once.
17+
#
18+
# Deliberately NOT merge blocking, and deliberately not on `pull_request`.
19+
# `latest` is a moving external dependency: if it gated merges, one upstream
20+
# rename would turn every open PR red at the same time, including the PR that
21+
# fixes it. Schedule plus manual dispatch is the correct contract, and it is
22+
# why this row does not block the release even though it lives in the live
23+
# tier.
24+
#
25+
# CAPACITY: both servers want an accelerator to start, and this project has no
26+
# runner with one yet (#641). Until it does, `runs-on` has nothing to point
27+
# at, so the job is gated on a repository variable: it stays skipped (not
28+
# queued forever, not failing every week) until somebody sets
29+
# METRIC_DRIFT_RUNNER to the label of a GPU runner with the NVIDIA container
30+
# toolkit installed. Setting that variable is the only step needed to turn
31+
# this on. Opening the refresh PR also needs the repository setting "Allow
32+
# GitHub Actions to create and approve pull requests".
33+
name: Metric-name drift (SGLang, TGI)
34+
35+
on:
36+
schedule:
37+
# Mondays, early UTC. Weekly is enough: upstream releases are weeks apart
38+
# and the fixture only has to be fresher than our next release.
39+
- cron: '23 5 * * 1'
40+
workflow_dispatch:
41+
inputs:
42+
model:
43+
description: Model both servers load. Small and ungated; generation quality is irrelevant, only metric registration is.
44+
required: false
45+
default: Qwen/Qwen2.5-0.5B-Instruct
46+
47+
permissions:
48+
contents: write
49+
pull-requests: write
50+
51+
jobs:
52+
capture:
53+
# Never on forks (a fork has neither the runner nor anywhere useful to
54+
# send the PR), and never before a runner exists (#641).
55+
if: github.repository == 'kubernetes-sigs/inference-perf' && vars.METRIC_DRIFT_RUNNER != ''
56+
runs-on: ${{ vars.METRIC_DRIFT_RUNNER }}
57+
timeout-minutes: 60
58+
# One job, one server, one pull request: a TGI rename must not be held up
59+
# behind an SGLang server that failed to boot.
60+
strategy:
61+
fail-fast: false
62+
matrix:
63+
include:
64+
- server: sglang
65+
image: docker.io/lmsysorg/sglang:latest
66+
port: 30000
67+
# SGLang exports nothing at all without --enable-metrics. The
68+
# model flag comes last so the model name is appended as its own
69+
# argument, never spliced into this string.
70+
launch_args: python3 -m sglang.launch_server --host 0.0.0.0 --port 30000 --enable-metrics --model-path
71+
warmup_body: '{"text":"1 2 3","sampling_params":{"max_new_tokens":4}}'
72+
- server: tgi
73+
image: ghcr.io/huggingface/text-generation-inference:latest
74+
port: 80
75+
launch_args: --model-id
76+
warmup_body: '{"inputs":"1 2 3","parameters":{"max_new_tokens":4}}'
77+
env:
78+
MODEL: ${{ inputs.model || 'Qwen/Qwen2.5-0.5B-Instruct' }}
79+
SERVER: ${{ matrix.server }}
80+
FIXTURE: e2e/testdata/server_metric_families/${{ matrix.server }}.txt
81+
steps:
82+
- name: Checkout code
83+
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
84+
85+
- name: Set up Python
86+
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
87+
with:
88+
python-version: '3.13'
89+
90+
- name: Set up PDM
91+
uses: pdm-project/setup-pdm@c6081998a653693a457d93c70b8007d979bc6741 # v4
92+
with:
93+
python-version: '3.13'
94+
95+
- name: Install dependencies
96+
run: pdm sync -d
97+
98+
- name: Start ${{ matrix.server }}
99+
env:
100+
IMAGE: ${{ matrix.image }}
101+
PORT: ${{ matrix.port }}
102+
LAUNCH_ARGS: ${{ matrix.launch_args }}
103+
run: |
104+
set -euo pipefail
105+
106+
# GPU runners are usually persistent, so never inherit a container
107+
# from a previous run.
108+
docker rm -f drift 2>/dev/null || true
109+
docker pull "${IMAGE}"
110+
111+
# LAUNCH_ARGS is unquoted on purpose so it word-splits into
112+
# arguments; it is a literal from this workflow file, not input.
113+
docker run -d --name drift --gpus all --shm-size 8g \
114+
-p "127.0.0.1:8080:${PORT}" \
115+
-v "${HOME}/hf-cache:/data" -e HF_HOME=/data \
116+
"${IMAGE}" ${LAUNCH_ARGS} "${MODEL}"
117+
118+
# Both images do a model download plus an engine warmup on first
119+
# boot, so this deadline is generous. It is also what stands between
120+
# a wedged boot and the job timeout.
121+
for _ in $(seq 1 120); do
122+
if curl -sf http://127.0.0.1:8080/health > /dev/null; then
123+
echo "${SERVER} is ready"
124+
exit 0
125+
fi
126+
sleep 10
127+
done
128+
echo "::error::${SERVER} did not become ready"
129+
exit 1
130+
131+
# Both servers register most metric families lazily, on first use, so a
132+
# scrape of a freshly booted server under-reports. One real request is
133+
# enough; no benchmark run is needed.
134+
- name: Warm up ${{ matrix.server }}
135+
env:
136+
WARMUP_BODY: ${{ matrix.warmup_body }}
137+
run: |
138+
set -euo pipefail
139+
curl -sf -X POST http://127.0.0.1:8080/generate \
140+
-H 'Content-Type: application/json' \
141+
-d "${WARMUP_BODY}" > /dev/null
142+
143+
# Writes only when the family list or the reported version differs from
144+
# what is committed, so an unchanged metric surface leaves no diff and
145+
# the step below has nothing to open.
146+
- name: Capture metric-family fixture
147+
run: |
148+
set -euo pipefail
149+
pdm run python scripts/capture_server_metric_names.py \
150+
--server "${SERVER}" --base-url http://127.0.0.1:8080
151+
152+
- name: Collect server logs
153+
if: always()
154+
run: docker logs drift > "${SERVER}-server.log" 2>&1 || true
155+
156+
- name: Upload server logs
157+
if: always()
158+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
159+
with:
160+
name: ${{ matrix.server }}-server-log
161+
path: ${{ matrix.server }}-server.log
162+
163+
- name: Stop ${{ matrix.server }}
164+
if: always()
165+
run: docker rm -f drift 2>/dev/null || true
166+
167+
# A refresh PR, not a test failure: the diff is the finding, and the
168+
# checks on the PR say whether it breaks anything we declare.
169+
- name: Open a fixture-refresh pull request
170+
env:
171+
GH_TOKEN: ${{ github.token }}
172+
run: |
173+
set -euo pipefail
174+
175+
if git diff --quiet -- "${FIXTURE}"; then
176+
echo "${SERVER} metric names unchanged; nothing to do."
177+
exit 0
178+
fi
179+
180+
VERSION="$(sed -n 's/^# version: //p' "${FIXTURE}")"
181+
BRANCH="metric-drift/${SERVER}"
182+
183+
git config user.name 'github-actions[bot]'
184+
git config user.email '41898282+github-actions[bot]@users.noreply.github.com'
185+
git switch -c "${BRANCH}"
186+
git add "${FIXTURE}"
187+
git commit -m "test: refresh ${SERVER} metric-name fixture (${VERSION})"
188+
# Force push: the branch is a rolling snapshot of latest, so a
189+
# second drift before the first PR merges should update that PR
190+
# rather than stack a second one.
191+
git push --force origin "${BRANCH}"
192+
193+
if gh pr view "${BRANCH}" --json state --jq .state 2>/dev/null | grep -q OPEN; then
194+
echo "Existing PR for ${BRANCH} updated in place."
195+
exit 0
196+
fi
197+
198+
cat > pr-body.md <<EOF
199+
Automated by .github/workflows/metric_name_drift.yml (#669).
200+
201+
The /metrics exposition of ${SERVER} ${VERSION} no longer matches the checked-in
202+
fixture. The diff is the drift.
203+
204+
If a name inference-perf declares in get_prometheus_metric_metadata() disappeared,
205+
e2e/tests/test_sglang_tgi_metric_names.py fails on this PR, and the declaration in
206+
inference_perf/client/modelserver/${SERVER}_client.py needs updating in the same
207+
change. If the checks are green, upstream added or removed metrics this project does
208+
not query and the refresh can merge as is.
209+
EOF
210+
211+
gh pr create \
212+
--base main --head "${BRANCH}" \
213+
--title "test: refresh ${SERVER} metric-name fixture (${VERSION})" \
214+
--body-file pr-body.md

‎e2e/testdata/server_metric_families/sglang.txt‎

Lines changed: 180 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,180 @@
1+
# sglang metric families, as inference-perf expects to find them on /metrics.
2+
#
3+
# provenance: upstream-source
4+
# server: sglang
5+
# version: v0.5.17
6+
# source: https://github.com/sgl-project/sglang/tree/v0.5.17/python/sglang (metric registrations, chiefly srt/observability/metrics_collector.py)
7+
# captured: 2026-08-17
8+
#
9+
# Regenerate against a running server:
10+
# python scripts/capture_server_metric_names.py --server sglang \
11+
# --base-url http://127.0.0.1:<port>
12+
#
13+
# UNVERIFIED AGAINST A LIVE SERVER. SGLang needs an accelerator to start, and
14+
# inference-perf has no runner with one yet (#641), so this snapshot was derived
15+
# by reading the pinned upstream registration code instead of by scraping a
16+
# running server. Derivation rule, reproducible by hand: every Gauge, Counter,
17+
# Histogram, Summary and GaugeHistogram construction under python/sglang/ at tag
18+
# v0.5.17 whose name literal starts with "sglang:". GaugeHistogram wraps a
19+
# prometheus_client Gauge (python/sglang/srt/utils/gauge_histogram.py), so it is
20+
# recorded as a gauge. prometheus_client's plain-text exposition appends _total
21+
# to counter family names; every SGLang counter already ends in _total, so the
22+
# registered name and the exposed family name coincide here.
23+
#
24+
# Two consequences of source derivation, both of which make this file a strict
25+
# SUPERSET of any single live exposition:
26+
# - families gated on optional features (disaggregation, hicache, LoRA, spec
27+
# decoding, grammar) are listed even though a default server never
28+
# registers them;
29+
# - metrics-registration is lazy in places, so a freshly started server
30+
# exposes fewer families than it can register.
31+
# Anything asserting family-for-family equality against a live server must
32+
# therefore wait for a live-scrape regeneration. Run the scheduled workflow
33+
# (.github/workflows/metric_name_drift.yml) once a runner exists, or run
34+
# scripts/capture_server_metric_names.py by hand against your own server, and
35+
# commit the result; provenance flips to live-scrape and the equality check
36+
# stops skipping.
37+
sglang:backup_bandwidth histogram
38+
sglang:backup_pgs histogram
39+
sglang:backuped_tokens_total counter
40+
sglang:cache_hit_rate gauge
41+
sglang:cached_tokens_total counter
42+
sglang:context_len gauge
43+
sglang:cuda_graph_passes_total counter
44+
sglang:decode_sum_seq_lens gauge
45+
sglang:dp_cooperation_forward_execution_seconds_total counter
46+
sglang:dp_cooperation_realtime_tokens_total counter
47+
sglang:e2e_request_latency_seconds histogram
48+
sglang:encoder_cache_entries gauge
49+
sglang:encoder_cache_evictions_total counter
50+
sglang:encoder_cache_hit_files_total counter
51+
sglang:encoder_cache_hit_tokens_total counter
52+
sglang:encoder_cache_size_mb gauge
53+
sglang:encoder_cache_total_files_total counter
54+
sglang:encoder_cache_total_tokens_total counter
55+
sglang:encoder_dp_pending_requests gauge
56+
sglang:encoder_mm_items_per_batch histogram
57+
sglang:encoder_mm_items_per_request histogram
58+
sglang:encoder_model_forward_seconds histogram
59+
sglang:encoder_preprocess_seconds histogram
60+
sglang:encoder_queue_wait_seconds histogram
61+
sglang:encoder_request_e2e_latency_seconds histogram
62+
sglang:encoder_requests_received_total counter
63+
sglang:encoder_requests_total counter
64+
sglang:encoder_transfer_seconds histogram
65+
sglang:eplb_balancedness summary
66+
sglang:eplb_gpu_physical_count histogram
67+
sglang:estimated_flops_per_gpu_total counter
68+
sglang:estimated_read_bytes_per_gpu_total counter
69+
sglang:estimated_write_bytes_per_gpu_total counter
70+
sglang:evicted_tokens_total counter
71+
sglang:eviction_duration_seconds histogram
72+
sglang:failed_session_recoveries_total counter
73+
sglang:forward_execution_seconds_total counter
74+
sglang:full_token_usage gauge
75+
sglang:func_latency_seconds histogram
76+
sglang:fwd_occupancy gauge
77+
sglang:gen_throughput gauge
78+
sglang:generation_tokens_histogram histogram
79+
sglang:generation_tokens_total counter
80+
sglang:get_loads_duration_seconds histogram
81+
sglang:grammar_compilation_time_seconds histogram
82+
sglang:grammar_ebnf_size histogram
83+
sglang:grammar_schema_count histogram
84+
sglang:grammar_tree_traversal_time_avg histogram
85+
sglang:grammar_tree_traversal_time_max histogram
86+
sglang:graph_memory_usage_gb gauge
87+
sglang:hicache_host_total_tokens gauge
88+
sglang:hicache_host_used_tokens gauge
89+
sglang:http_requests_active gauge
90+
sglang:http_requests_total counter
91+
sglang:http_responses_total counter
92+
sglang:inter_token_latency_seconds histogram
93+
sglang:is_cuda_graph gauge
94+
sglang:kv_available_tokens gauge
95+
sglang:kv_cache_memory_usage_gb gauge
96+
sglang:kv_evictable_tokens gauge
97+
sglang:kv_transfer_alloc_ms histogram
98+
sglang:kv_transfer_bootstrap_ms histogram
99+
sglang:kv_transfer_latency_ms histogram
100+
sglang:kv_transfer_speed_gb_s histogram
101+
sglang:kv_transfer_total_mb histogram
102+
sglang:kv_used_tokens gauge
103+
sglang:load_back_duration_seconds histogram
104+
sglang:load_back_tokens_total counter
105+
sglang:lora_pool_slots_total gauge
106+
sglang:lora_pool_slots_used gauge
107+
sglang:lora_pool_utilization gauge
108+
sglang:mamba_available_tokens gauge
109+
sglang:mamba_evictable_tokens gauge
110+
sglang:mamba_usage gauge
111+
sglang:mamba_used_tokens gauge
112+
sglang:max_running_requests_under_SLO gauge
113+
sglang:max_total_num_tokens gauge
114+
sglang:max_total_num_tokens_swa gauge
115+
sglang:new_token_ratio gauge
116+
sglang:num_aborted_requests_total counter
117+
sglang:num_bootstrap_failed_reqs_total counter
118+
sglang:num_decode_prealloc_queue_reqs gauge
119+
sglang:num_decode_transfer_queue_reqs gauge
120+
sglang:num_grammar_aborted_total counter
121+
sglang:num_grammar_cache_hit_total counter
122+
sglang:num_grammar_queue_reqs gauge
123+
sglang:num_grammar_timeout_total counter
124+
sglang:num_grammar_total counter
125+
sglang:num_pages gauge
126+
sglang:num_paused_reqs gauge
127+
sglang:num_prefill_bootstrap_queue_reqs gauge
128+
sglang:num_prefill_inflight_queue_reqs gauge
129+
sglang:num_prefill_retries_total counter
130+
sglang:num_queue_reqs gauge
131+
sglang:num_requests_total counter
132+
sglang:num_retracted_input_tokens_total counter
133+
sglang:num_retracted_output_tokens_total counter
134+
sglang:num_retracted_reqs gauge
135+
sglang:num_retracted_requests_total counter
136+
sglang:num_running_reqs gauge
137+
sglang:num_so_requests_total counter
138+
sglang:num_streaming_sessions gauge
139+
sglang:num_transfer_failed_reqs_total counter
140+
sglang:num_unique_running_routing_keys gauge
141+
sglang:num_used_tokens gauge
142+
sglang:page_size gauge
143+
sglang:pending_prealloc_token_usage gauge
144+
sglang:per_stage_req_latency_seconds histogram
145+
sglang:prefetch_bandwidth histogram
146+
sglang:prefetch_pgs histogram
147+
sglang:prefetched_tokens_total counter
148+
sglang:prefill_delayer_outcomes_total counter
149+
sglang:prefill_delayer_wait_forward_passes histogram
150+
sglang:prefill_delayer_wait_seconds histogram
151+
sglang:process_cpu_seconds_total counter
152+
sglang:prompt_tokens_histogram histogram
153+
sglang:prompt_tokens_total counter
154+
sglang:queue_time_seconds histogram
155+
sglang:realtime_tokens_total counter
156+
sglang:routing_key_all_req_count gauge
157+
sglang:routing_key_running_req_count gauge
158+
sglang:routing_keys_active gauge
159+
sglang:spec_accept_length gauge
160+
sglang:spec_accept_rate gauge
161+
sglang:spec_block_accept_length gauge
162+
sglang:spec_cap_length gauge
163+
sglang:spec_num_draft_tokens gauge
164+
sglang:spec_num_steps gauge
165+
sglang:spec_verify_calls_total counter
166+
sglang:startup_available_gpu_memory_gb gauge
167+
sglang:startup_cuda_graph_time_seconds gauge
168+
sglang:startup_latency_breakdown_seconds_max gauge
169+
sglang:startup_time_seconds gauge
170+
sglang:streaming_session_held_tokens gauge
171+
sglang:swa_available_tokens gauge
172+
sglang:swa_evictable_tokens gauge
173+
sglang:swa_token_usage gauge
174+
sglang:swa_used_tokens gauge
175+
sglang:time_to_first_token_seconds histogram
176+
sglang:token_usage gauge
177+
sglang:uncached_prompt_tokens_histogram histogram
178+
sglang:utilization gauge
179+
sglang:weight_load_duration_seconds gauge
180+
sglang:weight_memory_usage_gb gauge

0 commit comments

Comments
 (0)