fix(backends): fold Claude cache tokens into prompt_tokens total

daydream/backends/__init__.py

@@ -105,11 +105,12 @@ class CostEvent:
             model is unknown to the table.
         input_tokens: Prompt tokens (None when unavailable).
         output_tokens: Completion tokens (None when unavailable).
-        cached_tokens: Cached portion of input_tokens (subset, NOT added
-            to input_tokens per D-15). None when unavailable. Default
-            ``None`` keeps existing 3-positional-arg call sites in
-            ``backends/claude.py`` and ``backends/codex.py`` valid until
-            Plans 03/04 update them.
+        cached_tokens: Cache-read hit subset of input_tokens. input_tokens
+            is the total input (backends fold cache read+creation into it);
+            cached_tokens is the read subset, NOT added to input_tokens.
+            None when unavailable. Default ``None`` keeps existing
+            3-positional-arg call sites in ``backends/claude.py`` and
+            ``backends/codex.py`` valid until Plans 03/04 update them.
         reasoning_tokens: Reasoning portion of output_tokens (subset, NOT
             additive — Codex's ``accounting.rs`` already counts these
             inside ``output_tokens``). Surfaces Codex's
@@ -156,9 +157,10 @@ class MetricsEvent:
             EVNT-02 (int, not Optional). Backends read the SDK key
             (Claude ``usage["output_tokens"]``, Codex
             ``usage["output_tokens"]``) and rename at the boundary.
-        cached_tokens: Subset of ``prompt_tokens`` served from cache
-            (None when unavailable). NOT additive to ``prompt_tokens``
-            (D-15).
+        cached_tokens: Cache-read hit subset of ``prompt_tokens``
+            (None when unavailable). ``prompt_tokens`` is the total input
+            (backends fold cache read+creation into it); cached_tokens is
+            the read subset, NOT additive to ``prompt_tokens``.
         cost_usd: Per-turn cost in USD (None when unavailable). Codex
             synthesizes via the #61 price table (#194 reverses D-16); None
             only when the model is unknown to the table.

daydream/backends/claude.py

@@ -82,6 +82,24 @@
 )
 
 
+def _total_input_tokens(usage: dict[str, Any]) -> int | None:
+    """Fold Anthropic's three input buckets into the true total input.
+
+    Anthropic reports `input_tokens` as the *uncached remainder* only, with
+    cache hits and writes split into `cache_read_input_tokens` and
+    `cache_creation_input_tokens`. These two cache buckets are not mutually
+    exclusive: a single response can both read one cache breakpoint and write
+    another, so both may be non-zero at once. ATIF's `Metrics.prompt_tokens`
+    is the total input, so sum `input_tokens`, `cache_read_input_tokens`, and
+    `cache_creation_input_tokens` whenever present. Returns None when
+    `input_tokens` is absent (preserves the no-token-count gate).
+    """
+    input_tokens = usage.get("input_tokens")
+    if input_tokens is None:
+        return None
+    return input_tokens + (usage.get("cache_read_input_tokens") or 0) + (usage.get("cache_creation_input_tokens") or 0)
+
+
 class ClaudeAgentError(Exception):
     """Raised when the Claude agent run reports an error result.
 
@@ -380,9 +398,11 @@ async def execute(
                             and msg_usage.get("input_tokens") is not None
                             and msg_usage.get("output_tokens") is not None
                         ):
+                            total_input = _total_input_tokens(msg_usage)
+                            assert total_input is not None  # guarded by input_tokens check above
                             yield MetricsEvent(
                                 message_id=getattr(msg, "message_id", "") or "",
-                                prompt_tokens=msg_usage["input_tokens"],
+                                prompt_tokens=total_input,
                                 completion_tokens=msg_usage["output_tokens"],
                                 cached_tokens=msg_usage.get("cache_read_input_tokens"),
                                 cost_usd=None,
@@ -415,14 +435,17 @@ async def execute(
                         if msg.structured_output is not None:
                             structured_result = msg.structured_output
                         # EVNT-04/05: emit CostEvent when cost OR usage is available.
-                        # Per-call semantics trusted for SDK 0.1.52 (D-14). cached_tokens
-                        # is a SUBSET of input_tokens, not additive (D-15).
+                        # Per-call semantics trusted for SDK 0.1.52 (D-14). Anthropic's raw
+                        # `input_tokens` is the *uncached remainder* only; we fold in the
+                        # cache-read and cache-creation buckets so the emitted value is the
+                        # true total input, matching ATIF Metrics.prompt_tokens. cached_tokens
+                        # stays the cache-read hit subset of that total.
                         result_usage = getattr(msg, "usage", None)
                         if msg.total_cost_usd is not None or result_usage is not None:
                             usage = result_usage or {}
                             yield CostEvent(
                                 cost_usd=msg.total_cost_usd,
-                                input_tokens=usage.get("input_tokens"),
+                                input_tokens=_total_input_tokens(usage),
                                 output_tokens=usage.get("output_tokens"),
                                 cached_tokens=usage.get("cache_read_input_tokens"),
                                 model_name=last_assistant_model,

daydream/pr_comment_renderer.py

@@ -339,7 +339,9 @@ def _accumulate_metrics(
     prompt = max(metrics.prompt_tokens or 0, 0)
     completion = max(metrics.completion_tokens or 0, 0)
     cached_raw = max(metrics.cached_tokens or 0, 0)
-    cached = min(cached_raw, prompt)  # ATIF: cached is a SUBSET of prompt
+    # defensive guard: backends emit cached ≤ prompt (cache reads folded into the
+    # total); clamp protects against malformed/legacy metrics
+    cached = min(cached_raw, prompt)
     phase.input_tokens += prompt
     phase.cached_tokens += cached
     phase.output_tokens += completion

daydream/trajectory.py

@@ -557,8 +557,10 @@ def _dispatch(self, event: Any) -> None:
                     result=ObservationResult(source_call_id=event.id, content=event.output),
                 )
         elif isinstance(event, MetricsEvent):
-            # EVNT-02 attribute names verbatim (D-15: cached_tokens is a
-            # SUBSET of prompt_tokens, not added).
+            # EVNT-02 attribute names verbatim. prompt_tokens is the total
+            # input (backends fold cache read+creation into it); cached_tokens
+            # is the cache-read hit subset (a subset of prompt_tokens, not
+            # added).
             #
             # D-04 correlation fallback (Codex): Codex emits no per-message
             # id, so MetricsEvent.message_id is always '' on the Codex path.

tests/test_backend_claude_metrics.py

@@ -138,7 +138,7 @@ async def test_dropped_token_bug_fixed(monkeypatch):
         ],
     )
     cost = next(e for e in events if isinstance(e, CostEvent))
-    assert cost.input_tokens == 100
+    assert cost.input_tokens == 130  # 100 uncached + 30 read folded into the total
     assert cost.output_tokens == 50
     assert cost.cached_tokens == 30
     assert cost.cost_usd == 0.001
@@ -162,15 +162,123 @@ async def test_metrics_event_emitted_per_assistant_message(monkeypatch):
     assert len(metrics) == 1
     m = metrics[0]
     assert m.message_id == "msg_01"
-    assert m.prompt_tokens == 100  # EVNT-02 verbatim name (NOT input_tokens)
+    assert m.prompt_tokens == 130  # EVNT-02 verbatim name; 100 uncached + 30 read folded in
     assert m.completion_tokens == 50  # EVNT-02 verbatim name (NOT output_tokens)
     assert m.cached_tokens == 30
     assert m.cost_usd is None  # AssistantMessage carries no per-message cost
 
 
 @pytest.mark.asyncio
-async def test_cached_tokens_is_subset_not_additive(monkeypatch):
-    """D-15: cached_tokens is reported alongside, NOT added to prompt_tokens."""
+async def test_prompt_tokens_include_cache_read_and_creation(monkeypatch):
+    """prompt_tokens folds input + cache_read + cache_creation into the true total input."""
+    # Fully-cached turn: raw input_tokens is the uncached remainder (22); the total
+    # input the model actually processed is 22 + 20000 read = 20022.
+    events = await _collect_events(
+        monkeypatch,
+        [
+            MockAssistantMessageWithUsage(
+                content=[MockTextBlock(text="cached")],
+                message_id="msg_cached",
+                usage={
+                    "input_tokens": 22,
+                    "output_tokens": 100,
+                    "cache_read_input_tokens": 20000,
+                    "cache_creation_input_tokens": 0,
+                },
+            ),
+            MockResultMessageWithUsage(
+                total_cost_usd=0.002,
+                structured_output=None,
+                usage={
+                    "input_tokens": 22,
+                    "output_tokens": 100,
+                    "cache_read_input_tokens": 20000,
+                    "cache_creation_input_tokens": 0,
+                },
+            ),
+        ],
+    )
+    metrics = [e for e in events if isinstance(e, MetricsEvent)][0]
+    cost = [e for e in events if isinstance(e, CostEvent)][0]
+    assert metrics.prompt_tokens == 20022
+    assert metrics.cached_tokens == 20000
+    assert metrics.completion_tokens == 100
+    assert cost.input_tokens == 20022
+    assert cost.cached_tokens == 20000
+
+    # Cache-write turn: creation tokens fold in too; a write is not a read hit, so
+    # cached_tokens stays 0.
+    events = await _collect_events(
+        monkeypatch,
+        [
+            MockAssistantMessageWithUsage(
+                content=[MockTextBlock(text="write")],
+                message_id="msg_write",
+                usage={
+                    "input_tokens": 50,
+                    "output_tokens": 100,
+                    "cache_read_input_tokens": 0,
+                    "cache_creation_input_tokens": 15000,
+                },
+            ),
+            MockResultMessageWithUsage(
+                total_cost_usd=0.003,
+                structured_output=None,
+                usage={
+                    "input_tokens": 50,
+                    "output_tokens": 100,
+                    "cache_read_input_tokens": 0,
+                    "cache_creation_input_tokens": 15000,
+                },
+            ),
+        ],
+    )
+    metrics = [e for e in events if isinstance(e, MetricsEvent)][0]
+    cost = [e for e in events if isinstance(e, CostEvent)][0]
+    assert metrics.prompt_tokens == 15050
+    assert metrics.cached_tokens == 0
+    assert cost.input_tokens == 15050
+    assert cost.cached_tokens == 0
+
+    # Read-and-write turn: the buckets are not mutually exclusive. One breakpoint
+    # is read (18000) while another is written (12000); both fold into the total
+    # (40 + 18000 + 12000 = 30040), but cached_tokens reflects only the read hit.
+    events = await _collect_events(
+        monkeypatch,
+        [
+            MockAssistantMessageWithUsage(
+                content=[MockTextBlock(text="both")],
+                message_id="msg_both",
+                usage={
+                    "input_tokens": 40,
+                    "output_tokens": 100,
+                    "cache_read_input_tokens": 18000,
+                    "cache_creation_input_tokens": 12000,
+                },
+            ),
+            MockResultMessageWithUsage(
+                total_cost_usd=0.004,
+                structured_output=None,
+                usage={
+                    "input_tokens": 40,
+                    "output_tokens": 100,
+                    "cache_read_input_tokens": 18000,
+                    "cache_creation_input_tokens": 12000,
+                },
+            ),
+        ],
+    )
+    metrics = [e for e in events if isinstance(e, MetricsEvent)][0]
+    cost = [e for e in events if isinstance(e, CostEvent)][0]
+    assert metrics.prompt_tokens == 30040
+    assert metrics.cached_tokens == 18000
+    assert cost.input_tokens == 30040
+    assert cost.cached_tokens == 18000
+
+
+@pytest.mark.asyncio
+async def test_prompt_tokens_is_total_of_all_input_buckets(monkeypatch):
+    """prompt_tokens is the total (uncached + read + creation); cached_tokens is the read subset."""
     events = await _collect_events(
         monkeypatch,
         [
@@ -188,9 +296,9 @@ async def test_cached_tokens_is_subset_not_additive(monkeypatch):
     )
     metrics = [e for e in events if isinstance(e, MetricsEvent)][0]
     cost = [e for e in events if isinstance(e, CostEvent)][0]
-    assert metrics.prompt_tokens == 500  # NOT 800 (D-15: cached is a subset)
+    assert metrics.prompt_tokens == 800  # 500 uncached + 300 read
     assert metrics.cached_tokens == 300
-    assert cost.input_tokens == 500  # CostEvent boundary keeps SDK names
+    assert cost.input_tokens == 800
     assert cost.cached_tokens == 300
 
 

tests/test_pr_comment_integration.py

@@ -424,14 +424,15 @@ async def test_per_phase_rollup_distinguishes_phases(
     review_row = _phase_row(markdown, "Review")
     parse_row = _phase_row(markdown, "Parse Feedback")
 
-    # Review used 4,000 input tokens → row should display "4,000".
-    # Parse Feedback used 1,000 input tokens → row should display "1,000".
-    assert "4,000" in review_row, (
-        f"Bug B/C: Review row missing real input tokens (expected 4,000).\n"
+    # prompt_tokens is the total input (uncached remainder + cache read folded in).
+    # Review: 4,000 + 1,500 read = 5,500 → row should display "5,500".
+    # Parse Feedback: 1,000 + 500 read = 1,500 → row should display "1,500".
+    assert "5,500" in review_row, (
+        f"Bug B/C: Review row missing real input tokens (expected 5,500 total).\n"
         f"  row: {review_row!r}\n  per-step metrics: {per_step_metrics}"
     )
-    assert "1,000" in parse_row, (
-        f"Bug B/C: Parse Feedback row missing real input tokens (expected 1,000).\n"
+    assert "1,500" in parse_row, (
+        f"Bug B/C: Parse Feedback row missing real input tokens (expected 1,500 total).\n"
         f"  row: {parse_row!r}\n  per-step metrics: {per_step_metrics}"
     )
     # And costs should differ — Review $0.20 vs Parse $0.05.

tests/test_pr_comment_renderer.py

@@ -616,9 +616,37 @@ def test_e2e_corrupted_trajectory_falls_back() -> None:
     assert "<sub>Generated by daydream v" in out
 
 
+# Regression lock: cached turn renders true total input (fix landed in the
+# Claude backend fold; this locks the renderer's total-form display contract).
+def test_cached_turn_renders_total_input(tmp_path: Path) -> None:
+    """A cached Claude turn renders its total input, not the uncached remainder.
+
+    The backend folds cache read/creation into prompt_tokens (the ATIF total),
+    so a step with prompt=20000, cached=15000 must render the true total input
+    with an honest read/total hit ratio.
+    """
+    p = _write_trajectory(
+        tmp_path,
+        steps=[
+            _user_step(),
+            _agent_step(
+                step_id=2,
+                phase="review",
+                model="claude-sonnet-4-5",
+                prompt=20000,
+                completion=800,
+                cached=15000,
+                cost_usd=0.1,
+            ),
+        ],
+    )
+    out = render_run_info_block([p])
+    assert "20,000 in (15,000 cached, 75% hit) → 800 out" in out
+
+
 # Regression: corrupt-metrics bleed (CodeRabbit #3 on PR #66)
 def test_metrics_clamped_when_cached_exceeds_prompt(tmp_path: Path) -> None:
-    """cached_tokens > prompt_tokens must be clamped to prompt at aggregation.
+    """malformed input where cached > prompt is defensively clamped.
 
     Per ATIF v1.6, cached_tokens is a SUBSET of prompt_tokens. A trajectory
     that reports prompt=10, cached=20 (corrupt or racy upstream) must not