test: resolve declared metric names by type, not by name alone

The live checks decided a declared name was usable by testing series presence
alone. A family that keeps its name and changes type passes that test while the
query built for it returns nonsense: increase() over a gauge does not raise, it
reports a wrong number. That is the same failure shape as #669 itself, where the
check verified the server's naming rather than the query's selection.

Both live checks now run two oracles over one exposition, each driven by the
metric's own candidate_names(): is_exposed for series presence, resolves for the
type the query assumes. Failures report absent and wrong_type separately because
they are different bugs with different fixes. in_golden becomes resolves, since
it now takes any family -> type map rather than a golden specifically.

Also drops vllm:prompt_tokens_recomputed from CONDITIONALLY_EXPOSED: vllm_client
no longer declares it and vLLM registers it nowhere in v0.26.0 through v0.28.0,
so it was shrinking the strict checks for nothing.
test_conditionally_exposed_still_apply guards that list the way
test_known_unresolved_still_do_not_resolve already guards the other one.

Checked against latest vLLM while here: all 33 declared names resolve against
v0.28.0, whose metric registrations are byte-identical to v0.26.0.

Author: Bslabe123

e2e/tests/test_sglang_tgi_metric_names.py

@@ -212,13 +212,26 @@ def test_exposed_families_match_fixture(server: str) -> None:
 
 @pytest.mark.parametrize("server", SERVER_IDS)
 def test_declared_metric_names_exist(server: str) -> None:
+    # Two oracles over one exposition, both driven by the metric's own
+    # candidate_names(): every series a query selects must be present
+    # (is_exposed, over sample and family names), and the family behind it must
+    # carry the type the query assumes (resolves, over the `# TYPE` map).
+    # Presence alone is not enough. Given `# TYPE sglang:prefix_cache_hit gauge`
+    # and a matching sample line, a declared CounterMetric("sglang:prefix_cache_hit")
+    # is_exposed -> True on the bare-name candidate group, while the increase()
+    # it emits is nonsense over a gauge and reports no error. resolves -> False.
     spec = SERVERS[server]
-    names = exposed_names(live_metrics_text(spec))
+    exposition = live_metrics_text(spec)
+    names = exposed_names(exposition)
+    families = parse_exposition(exposition, spec.prefix)
     declared = declared_for(spec)
     allowed = KNOWN_UNRESOLVED[server]
-
-    missing = sorted(name for name, metric in declared.items() if name not in allowed and not is_exposed(metric, names))
-    assert not missing, (
-        f"{len(missing)}/{len(declared)} names declared by the {server} client are absent from a real "
-        f"/metrics exposition (stale names produce silently empty report fields): {missing}"
+    checked = {name: metric for name, metric in declared.items() if name not in allowed}
+
+    absent = {name for name, metric in checked.items() if not is_exposed(metric, names)}
+    mistyped = sorted(name for name, metric in checked.items() if name not in absent and not resolves(metric, families))
+    assert not (absent or mistyped), (
+        f"{len(absent) + len(mistyped)}/{len(checked)} names declared by the {server} client are unusable "
+        f"against a real /metrics exposition (a stale name reports an empty field, a retyped family reports "
+        f"a nonsense one, and neither raises): absent={sorted(absent)}, wrong_type={mistyped}"
     )

e2e/tests/test_vllm_cpu_metric_names.py

@@ -31,7 +31,8 @@
    release, which is the deliberate state of ``latest``: a floating tag
    would guarantee golden rot, so it never gets one.
 3. ``test_declared_metric_names_exist`` (live, every release in the table):
-   declared names present in the real exposition, the end-invariant itself.
+   declared names present in the real exposition under the type their query
+   assumes, the end-invariant itself.
    On ``latest`` this is the early warning that an upstream rename is
    coming, red on live PRs before, not at, the next pin bump.
 
@@ -58,8 +59,8 @@
     exposed_vllm_families,
     format_golden,
     golden_path,
-    in_golden,
     is_exposed,
+    resolves,
     load_golden,
 )
 from utils.net import get_free_port
@@ -82,19 +83,22 @@
 # name declarations are read from it, never the tokenizer itself.
 GEMMA_TARBALL = "e2e/testdata/models/google_gemma-3-270m.tar.gz"
 
-# Declared names that a STOCK vLLM does not expose. All five arrived in #348
+# Declared names that a STOCK vLLM does not expose. All four arrived in #348
 # ("vLLM latest (0.15.0) production metrics") and are absent from a default
-# v0.26.0 server, seemingly gated on optional features (KV offloading and
-# similar) whose components never register their metric families on a stock
-# configuration. Kept out of the strict checks rather than deleted so the
+# v0.26.0 server, gated on optional features (VLLM_COMPUTE_NANS_IN_LOGITS and
+# --kv-cache-metrics) whose components never register their metric families on
+# a stock configuration. Kept out of the strict checks rather than deleted so the
 # declarations can be triaged: each is either config-gated (then this list
 # documents the gate) or stale (then it should be removed from vllm_client).
+#
+# Guarded by test_conditionally_exposed_still_apply. A fifth entry,
+# vllm:prompt_tokens_recomputed, outlived its declaration and sat here shrinking
+# the strict checks for nothing; it is registered nowhere in vLLM v0.26.0-v0.28.0.
 CONDITIONALLY_EXPOSED = {
     "vllm:corrupted_requests",
     "vllm:kv_block_idle_before_evict_seconds",
     "vllm:kv_block_lifetime_seconds",
     "vllm:kv_block_reuse_gap_seconds",
-    "vllm:prompt_tokens_recomputed",
 }
 
 # Declarations whose queries select nothing, with a fix already in flight. Kept
@@ -153,7 +157,7 @@ def test_declared_names_resolve_against_goldens(golden_file: Path) -> None:
     declared = _declared("http://127.0.0.1:1", DEFAULT_MODEL)
     assert declared, "vLLM client declared no metric names"
 
-    missing = sorted(name for name, metric in declared.items() if name not in SKIPPED and not in_golden(metric, golden))
+    missing = sorted(name for name, metric in declared.items() if name not in SKIPPED and not resolves(metric, golden))
     assert not missing, (
         f"{len(missing)}/{len(declared)} declared metric names do not resolve against {golden_file.name} "
         f"(stale names produce silently empty report fields): {missing}"
@@ -172,13 +176,36 @@ def test_known_unresolved_still_do_not_resolve(golden_file: Path) -> None:
     undeclared = sorted(name for name in KNOWN_UNRESOLVED if name not in declared)
     assert not undeclared, f"no longer declared, drop the KNOWN_UNRESOLVED entries: {undeclared}"
 
-    now_resolving = sorted(name for name in KNOWN_UNRESOLVED if in_golden(declared[name], golden))
+    now_resolving = sorted(name for name in KNOWN_UNRESOLVED if resolves(declared[name], golden))
     assert not now_resolving, (
         f"{now_resolving} now resolve against {golden_file.name}; drop their KNOWN_UNRESOLVED "
         f"entries so the strict check covers them again"
     )
 
 
+@pytest.mark.parametrize("golden_file", GOLDEN_FILES, ids=lambda p: p.stem)
+def test_conditionally_exposed_still_apply(golden_file: Path) -> None:
+    # Guards the other allowlist the same way. CONDITIONALLY_EXPOSED shrinks the
+    # strict checks, so each entry has to still be earning that: still declared by
+    # vllm_client, and still absent from a stock server's golden. Feeding it
+    # {"vllm:corrupted_requests"} against v0.26.0.txt passes, because the client
+    # declares it and the golden (captured from a stock server, which does not run
+    # with VLLM_COMPUTE_NANS_IN_LOGITS) does not list it. Dropping the declaration
+    # or a release starting to expose it both fail here, which is what stops the
+    # list quietly widening the hole it opens.
+    golden = load_golden(golden_file)
+    declared = _declared("http://127.0.0.1:1", DEFAULT_MODEL)
+
+    undeclared = sorted(name for name in CONDITIONALLY_EXPOSED if name not in declared)
+    assert not undeclared, f"no longer declared, drop the CONDITIONALLY_EXPOSED entries: {undeclared}"
+
+    now_resolving = sorted(name for name in CONDITIONALLY_EXPOSED if resolves(declared[name], golden))
+    assert not now_resolving, (
+        f"{now_resolving} resolve against {golden_file.name}, so they are not gated off on a stock "
+        f"server; drop their CONDITIONALLY_EXPOSED entries so the strict check covers them again"
+    )
+
+
 @pytest.mark.asyncio
 @pytest.mark.skipif(not VLLMServerRunner.is_available(), reason="no vLLM server or executable available")
 async def test_exposed_families_match_golden() -> None:
@@ -211,13 +238,29 @@ async def test_exposed_families_match_golden() -> None:
 @pytest.mark.asyncio
 @pytest.mark.skipif(not VLLMServerRunner.is_available(), reason="no vLLM server or executable available")
 async def test_declared_metric_names_exist() -> None:
+    # Two oracles over one exposition, both driven by the metric's own
+    # candidate_names(): every series a query selects must be present
+    # (is_exposed, over sample and family names), and the family behind it must
+    # carry the type the query assumes (resolves, over the `# TYPE` map).
+    # Presence alone is not enough, and that gap is the same shape as the bug
+    # this module exists for. Given `# TYPE vllm:prefix_cache_hits gauge` and a
+    # sample line `vllm:prefix_cache_hits{...} 3`, a declared
+    # CounterMetric("vllm:prefix_cache_hits") is_exposed -> True on the bare-name
+    # candidate group, while the query it emits, increase(...[60s]), is nonsense
+    # over a gauge and reports no error. resolves -> False catches it.
     async with VLLMServerRunner(port=get_free_port()) as server:
-        names = exposed_names(await _warmed_up_exposition(server))
+        exposition = await _warmed_up_exposition(server)
         declared = _declared(server.base_url, server.model)
 
     assert declared, "vLLM client declared no metric names"
-    missing = sorted(name for name, metric in declared.items() if name not in SKIPPED and not is_exposed(metric, names))
-    assert not missing, (
-        f"{len(missing)}/{len(declared)} declared metric names absent from a real vLLM /metrics exposition "
-        f"(stale names produce silently empty report fields): {missing}"
+    names = exposed_names(exposition)
+    families = exposed_vllm_families(exposition)
+    checked = {name: metric for name, metric in declared.items() if name not in SKIPPED}
+
+    absent = {name for name, metric in checked.items() if not is_exposed(metric, names)}
+    mistyped = sorted(name for name, metric in checked.items() if name not in absent and not resolves(metric, families))
+    assert not (absent or mistyped), (
+        f"{len(absent) + len(mistyped)}/{len(checked)} declared metric names unusable against a real vLLM "
+        f"/metrics exposition (a stale name reports an empty field, a retyped family reports a nonsense one, "
+        f"and neither raises): absent={sorted(absent)}, wrong_type={mistyped}"
     )

e2e/utils/metric_families.py

@@ -111,15 +111,26 @@ def is_exposed(metric: Metric[Any], names: Set[str]) -> bool:
     An exposition lists real series names, so this is a plain subset test over the
     metric's own candidate groups; nothing here needs to know how a counter or a
     histogram is spelled.
+
+    Presence only. It cannot see a family that kept its name and changed type, so
+    the live check pairs it with ``resolves`` over the same exposition's family
+    map rather than using it alone (#669).
     """
     return any(group <= names for group in metric.candidate_names())
 
 
-def in_golden(metric: Metric[Any], golden: Dict[str, str]) -> bool:
-    """Whether every series this metric's queries select resolves against a golden."""
+def resolves(metric: Metric[Any], families: Dict[str, str]) -> bool:
+    """Whether every series this metric's queries select resolves against a family map.
+
+    Takes a family -> type map, so it serves both a committed golden and a map
+    parsed off a live exposition. Unlike ``is_exposed`` this is type-aware: a
+    family carrying the right name under the wrong type does not resolve, which
+    is what a name-only check cannot see (a counter turned gauge still answers
+    to its bare name while ``increase()`` over it returns nonsense).
+    """
     metric_type = prometheus_type(metric)
     return any(
-        all(provided_by_families(series, metric_type, golden) for series in group) for group in metric.candidate_names()
+        all(provided_by_families(series, metric_type, families) for series in group) for group in metric.candidate_names()
     )
 
 

e2e/utils/server_metric_names.py

@@ -243,6 +243,10 @@ def is_exposed(metric: Metric[Any], names: Set[str]) -> bool:
 
     An exposition lists real series names, so this is a plain subset test over the
     metric's candidate groups.
+
+    Presence only. It cannot see a family that kept its name and changed type, so
+    the live check pairs it with ``resolves`` over the same exposition's family
+    map rather than using it alone (#669).
     """
     return any(group <= names for group in metric.candidate_names())