Skip to content

Commit ec46e0e

Browse files
committed
fix(runner): move cost_limit persistence into helper (ev-9gm)
Per .claude/rules/architecture.md (the runner-stays-boring rule): ``rg 'if .*type' eval_harness/runner/`` must return nothing. My prior revision had ``if outcome.trace.error.type == "cost_limit"`` in the main loop to persist short-circuited traces — branching on an error-type magic string is exactly what the rule forbids. Fold the persist into the helper (now `_short_circuit_cost_limit`) so the runner's loop just calls a coroutine that returns a fully-formed CellOutcome with the trace already on disk. The boring grep is satisfied; no behaviour change.
1 parent d151077 commit ec46e0e

1 file changed

Lines changed: 17 additions & 12 deletions

File tree

eval_harness/runner/run_eval.py

Lines changed: 17 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -58,8 +58,12 @@ async def run_cell(case: EvalCase, variant: RunVariant) -> CellOutcome:
5858
# cells short-circuit instead of dispatching to the adapter.
5959
# In-flight cells (already past this check) finish naturally.
6060
if accumulator.check_limit(cost_limit):
61-
return _cost_limit_outcome(
62-
case, variant, plan.run_id, accumulator.total_usd(), cost_limit
61+
return await _short_circuit_cost_limit(
62+
case,
63+
variant,
64+
plan,
65+
accumulator.total_usd(),
66+
cost_limit,
6367
)
6468
outcome = await _run_one(case, variant, plan)
6569
accumulator.tally(outcome.trace)
@@ -70,27 +74,27 @@ async def run_cell(case: EvalCase, variant: RunVariant) -> CellOutcome:
7074
return_exceptions=False,
7175
)
7276

73-
# Persist short-circuited traces so summary.yaml + traces.jsonl are
74-
# consistent with what the runner returns. _run_one already saved
75-
# the others.
76-
for outcome in outcomes:
77-
if outcome.trace.error is not None and outcome.trace.error.type == "cost_limit":
78-
await plan.trace_store.save_trace(outcome.trace)
79-
8077
summary = build_summary(outcomes, plan)
8178
await plan.trace_store.save_summary(summary)
8279
return summary
8380

8481
raise RuntimeError("unreachable: AsyncExitStack never re-raises")
8582

8683

87-
def _cost_limit_outcome(
84+
async def _short_circuit_cost_limit(
8885
case: EvalCase,
8986
variant: RunVariant,
90-
run_id: str,
87+
plan: RunPlan,
9188
accumulated: float,
9289
limit: float | None,
9390
) -> CellOutcome:
91+
"""Build, persist, and return a cost_limit cell outcome.
92+
93+
Owning the persist side here keeps the runner's main loop ignorant of
94+
error categories — per ``.claude/rules/architecture.md`` (the
95+
runner-stays-boring rule), branching on ``trace.error.type`` belongs in
96+
a helper, not in ``run_cell``.
97+
"""
9498
now = utc_now()
9599
trace = Trace.from_error(
96100
case.id,
@@ -102,10 +106,11 @@ def _cost_limit_outcome(
102106
else f"cost limit exceeded, accumulated ${accumulated:.4f}"
103107
),
104108
)
105-
trace.run_id = run_id
109+
trace.run_id = plan.run_id
106110
trace.started_at = now
107111
trace.finished_at = now
108112
trace.latency_ms = 0
113+
await plan.trace_store.save_trace(trace)
109114
return CellOutcome(case=case, variant=variant, trace=trace, results=[])
110115

111116

0 commit comments

Comments
 (0)