harness: offline-vs-live agreement gate + fix the discriminator itself

(GOTCHA #85)

Wires eval_policy_on_fixture.exs into eval_live_protocol.sh (10s, no
Dolphin, advisory, recorded in protocol.txt). Fixing it to be meaningful
required two changes: an offset SWEEP (0..8) replacing the delay-0-era
two-offset compare, and a --delay-id flag. ms_g6_sp1 goes from a
misleading 0.33 to 0.999 at offset 5 — its exact trained convention,
re-derived from the weights. Caveat recorded: teacher-forced, so it
cannot see closed-loop collapse (an untrained id still scores 0.93);
CycleSim is the complementary instrument.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SuqdnRgbvmFFduhyw3PUpy

Author: blasphemetheus

docs/reference/GOTCHAS.md

@@ -3300,3 +3300,23 @@ embeddable game state). Rule: EVERY human/pressure session that might
 feed training passes --replay-dir eval_runs/<name>/ at launch. The
 eval protocol does this implicitly (its r*.slp ARE the collected pool);
 ad-hoc play launches must do it explicitly.
+
+## 85. The offline fixture discriminator was delay-0-era: two offsets, no delay-id
+
+`scripts/eval_policy_on_fixture.exs` compared the policy's emission only
+against controller_N and controller_N+1, and constructed its Agent without
+a `delay_id`. Both assumptions predate the delay campaign. Consequence
+(measured 2026-08-03 while wiring it into the eval protocol): the two
+best policies in the repo — 380-434 shines/min live — scored ~0.33-0.40
+"agreement", which reads as "never learned the mapping" and would send a
+reader hunting a nonexistent training bug. With an offset SWEEP (0..8)
+and the trained id, ms_g6_sp1 scores **0.999 at offset 5** — exactly its
+trained `frame_delay 3 + pipeline_offset 2`. The argmax offset is now the
+useful output: it MEASURES the policy's effective label shift from the
+weights alone.
+
+Second lesson, recorded at the same time: this eval is TEACHER-FORCED on
+fixture states, so it cannot see closed-loop mode collapse. An UNTRAINED
+delay-id (catastrophic live: 71/min vs 434) still scores 0.93 offline.
+Offline agreement answers "did it learn the mapping"; CycleSim answers
+"does the closed loop hold". Use both.

scripts/eval_live_protocol.sh

@@ -113,6 +113,18 @@ for i in $(seq 1 "$RUNS"); do
   grep -a "Final stats\|Staleness\|skipped\|SD FAILED\|replay finalized" "$OUTDIR/r$i.log" | sed "s/^/  r$i /"
 done
 
+# Offline-vs-live discriminator (task #21, 2026-08-03). ~10s, no Dolphin.
+# Recorded BEFORE scoring so every block carries the number: high offline
+# agreement + poor live numbers => STATE-STREAM / delay problem, not a
+# learning failure (the distinction GOTCHA #81 took four escalations to
+# reach). Advisory, not fatal — we have no calibrated threshold yet, and a
+# hard gate on an uncalibrated number would block good runs.
+echo "=== offline fixture agreement"
+FIXTURE_AGREE=$(XLA_TARGET="${XLA_TARGET_EVAL:-cpu}" mix run scripts/eval_policy_on_fixture.exs \
+  --policy "$POLICY" 2>&1 | grep -a "FIXTURE_AGREEMENT" | tail -1)
+echo "  ${FIXTURE_AGREE:-(unavailable)}"
+echo "fixture_agreement: ${FIXTURE_AGREE:-unavailable}" >> "$OUTDIR/protocol.txt"
+
 echo "=== scoring"
 SLPS=$(ls "$OUTDIR"/r*.slp 2>/dev/null)
 if [ -z "$SLPS" ]; then echo "no replays captured" >&2; exit 4; fi

scripts/eval_policy_on_fixture.exs

@@ -27,7 +27,9 @@ alias ExPhil.Agents.Agent
 alias ExPhil.Training.Output
 
 {opts, _, _} =
-  OptionParser.parse(System.argv(), strict: [policy: :string, fixture: :string, port: :integer, limit: :integer])
+  OptionParser.parse(System.argv(),
+    strict: [policy: :string, fixture: :string, port: :integer, limit: :integer, delay_id: :integer]
+  )
 
 policy_path = opts[:policy] || raise "--policy required"
 fixture = opts[:fixture] || "test/fixtures/replays/fox_multishine_closed.slp"
@@ -49,7 +51,18 @@ Output.banner("Policy vs its own fixture (offline)")
 Output.puts("policy:  #{Path.basename(policy_path)}")
 Output.puts("fixture: #{Path.basename(fixture)} (#{length(frames)} frames)")
 
-{:ok, agent} = Agent.start_link(policy_path: policy_path, deterministic: true)
+# --delay-id N: delay-id policies have per-id behavioral MODES, and an
+# UNTRAINED id is catastrophic (2026-08-03: ms_g6_sp1 scores 434/min at a
+# trained id, 71 at an untrained one). Defaulting to 0 made this script
+# report ~0.40 agreement for two record-setting policies — a phantom
+# "training bug" that is really an untrained-mode measurement. Pass the id
+# the policy was trained on (LATENCY_ARCHITECTURE: sp1 => 2 or 3).
+{:ok, agent} =
+  Agent.start_link(
+    policy_path: policy_path,
+    deterministic: true,
+    delay_id: opts[:delay_id] || 0
+  )
 Agent.warmup(agent)
 
 # Compare against the input recorded on the SAME frame (delay 0) and on the
@@ -66,19 +79,43 @@ results =
   |> Enum.reject(&is_nil/1)
 
 n = length(results)
+arr = List.to_tuple(results)
+
+# Offset SWEEP (widened 2026-08-03). The original compared only offsets 0
+# and 1 — a delay-0-era assumption. Delay-trained policies emit the action
+# for frame N+d+pipeline_offset (measured intrinsic +2), so the champion
+# recipes land at offset ~4-5 and scored ~0.33 against the old two-offset
+# comparison, which reads as "never learned the mapping" when it is
+# actually "learned a different, correct convention". The argmax offset is
+# itself the useful output: it MEASURES the policy's effective label shift.
+offsets = 0..8
+
+agreements =
+  Map.new(offsets, fn off ->
+    pairs = max(n - off, 1)
+
+    hits =
+      Enum.count(0..(n - off - 1)//1, fn i ->
+        {pb, px, _, _} = elem(arr, i)
+        {_, _, ab, ax} = elem(arr, i + off)
+        pb == ab and px == ax
+      end)
+
+    {off, hits / pairs}
+  end)
 
-same_frame =
-  Enum.count(results, fn {pb, px, ab, ax} -> pb == ab and px == ax end)
-
-next_frame =
-  results
-  |> Enum.zip(tl(results) ++ [nil])
-  |> Enum.reject(fn {_a, b} -> is_nil(b) end)
-  |> Enum.count(fn {{pb, px, _, _}, {_, _, ab, ax}} -> pb == ab and px == ax end)
+{best_off, best} = Enum.max_by(agreements, fn {_off, a} -> a end)
 
 Output.puts("")
-Output.puts("B/X agreement vs controller_N   (delay 0): #{Float.round(100.0 * same_frame / max(n, 1), 1)}%")
-Output.puts("B/X agreement vs controller_N+1 (delay 1): #{Float.round(100.0 * next_frame / max(n - 1, 1), 1)}%")
+
+Output.puts(
+  "B/X agreement by offset: " <>
+    Enum.map_join(offsets, "  ", fn o ->
+      "#{o}=#{Float.round(100.0 * agreements[o], 1)}%"
+    end)
+)
+
+Output.puts("best offset #{best_off} (#{Float.round(100.0 * best, 1)}%)")
 
 # What the policy is actually pressing — a policy stuck on "B always" shows
 # up here instantly, and no agreement number explains that as clearly.
@@ -91,14 +128,23 @@ Output.puts("")
 Output.puts("press rates   policy: B=#{Float.round(pb_rate * 100, 1)}% X=#{Float.round(px_rate * 100, 1)}%")
 Output.puts("            fixture: B=#{Float.round(ab_rate * 100, 1)}% X=#{Float.round(ax_rate * 100, 1)}%")
 
-best = max(same_frame / max(n, 1), next_frame / max(n - 1, 1))
+# Machine-readable summary for the eval-protocol gate (task #21). Keep this
+# line's shape stable — eval_live_protocol.sh greps it.
+Output.puts(
+  "FIXTURE_AGREEMENT best=#{Float.round(best, 4)} offset=#{best_off} " <>
+    "delay_id=#{opts[:delay_id] || 0} n=#{n}"
+)
 
 cond do
   best > 0.95 ->
     Output.success(
-      "Policy reproduces the fixture offline. A live failure is then a STATE-STREAM " <>
-        "problem (parsed vs live action_frame, GOTCHAS #81) or an action_delay mismatch — " <>
-        "not a learning failure."
+      "Policy reproduces the fixture offline at offset #{best_off}. A live failure is " <>
+        "then a STATE-STREAM problem (parsed vs live action_frame, GOTCHAS #81), a " <>
+        "delay/id mismatch, or a CLOSED-LOOP failure this eval cannot see — it is " <>
+        "teacher-forced on fixture states, so mode collapse that only appears on " <>
+        "self-generated histories scores high here (measured: an UNTRAINED delay-id " <>
+        "still scores 0.93 offline while collapsing live). Use CycleSim for the " <>
+        "closed-loop question; the two instruments are complementary."
     )
 
   abs(pb_rate - ab_rate) > 0.2 or abs(px_rate - ax_rate) > 0.2 ->