Skip to content

Commit 8a9e1c8

Browse files
harness: tier-1 pins — af_convention default, dummy port, analog release
(a) af_convention must default to :parsed (GOTCHAS #81 SETTLED: the per-action delta varies per OCCURRENCE, so no key is correct). The two tests that pin the table's CONTENTS pass whether the feature is on or off, so a default flip would have passed the whole suite — they now carry deprecation banners pointing at this guard. (b) Extracted the dummy-port decision into ExPhil.Eval.PortCheck and unit-tested it (7 tests): HUMAN-where-CPU-expected, the level-1 autostart race, absent port, nil-level tolerance. This gate silently invalidated the entire combo-drill era and had zero tests. (c) ReplicationCheck :exact compared only DIGITAL l/r and was blind to the analog shoulder — the same tolerance-class blindness that let a latched analog RELEASE score 15/15 exact (#66). Now analog-aware (quantized 1/64); a latched release fails, identical streams pass, and the overfit-replication suite still passes exact. All three guards verified to fail when their invariant is broken. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01SuqdnRgbvmFFduhyw3PUpy
1 parent bdd385a commit 8a9e1c8

7 files changed

Lines changed: 309 additions & 40 deletions

File tree

lib/exphil/eval/port_check.ex

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
defmodule ExPhil.Eval.PortCheck do
2+
@moduledoc """
3+
Decide whether a replay's port setup matches what the run ASKED for.
4+
5+
Extracted from `scripts/check_replay_ports.exs` (2026-08-03) so the
6+
decision is unit-testable. It gates every CPU-dummy eval block, and until
7+
now had no tests at all — while the bug it guards silently invalidated an
8+
entire era of recordings.
9+
10+
The bug (GOTCHAS #57 / #57b): Slippi records each port's TYPE in the
11+
game-start block (0 HUMAN, 1 CPU, 2 DEMO, 3 empty), but Peppi's
12+
`PlayerMeta` does not expose it and libmelee never logged the achieved
13+
level. A dummy that silently came up HUMAN was therefore invisible — it
14+
just looked like a very passive CPU. On 2026-07-26, five of six recordings
15+
made with `--dummy cpu --dummy-cpu-level 9` produced a HUMAN port 2 (Wait
16+
76% of frames, never jumped, drifted onto the ledge); the tell was
17+
"level-9 CPUs don't ledge plank". Cause: an autostart race at character
18+
select. A fresh CPU also defaults to level 1, so a level mismatch is the
19+
same race firing before the slider drag finished.
20+
21+
Port maps use string keys because they arrive as decoded JSON from the
22+
py-slippi reader: `%{"port" => 2, "type" => 1, "cpu_level" => 9}`.
23+
"""
24+
25+
@type port_info :: %{optional(String.t()) => term()}
26+
@type verdict ::
27+
:ok
28+
| {:error, :absent, String.t()}
29+
| {:error, :not_cpu, String.t()}
30+
| {:error, :wrong_level, String.t()}
31+
32+
@human 0
33+
@cpu 1
34+
@demo 2
35+
@empty 3
36+
37+
@doc "Human-readable name for Slippi's port type code."
38+
@spec type_name(integer() | nil) :: String.t()
39+
def type_name(@human), do: "HUMAN"
40+
def type_name(@cpu), do: "CPU"
41+
def type_name(@demo), do: "DEMO"
42+
def type_name(@empty), do: "empty"
43+
def type_name(other), do: "type=#{inspect(other)}"
44+
45+
@doc """
46+
Verify `ports` against expectations.
47+
48+
Options:
49+
* `:expect_cpu` — port number that must be a CPU (nil = no check)
50+
* `:expect_level` — required CPU level (only checked when the replay
51+
reports one; some builds report nil)
52+
53+
Returns `:ok` or `{:error, reason_atom, message}`.
54+
"""
55+
@spec verify([port_info()], keyword()) :: verdict()
56+
def verify(ports, opts \\ []) do
57+
want_port = Keyword.get(opts, :expect_cpu)
58+
want_level = Keyword.get(opts, :expect_level)
59+
60+
cond do
61+
is_nil(want_port) ->
62+
:ok
63+
64+
true ->
65+
port = Enum.find(ports, &(&1["port"] == want_port))
66+
check_port(port, want_port, want_level)
67+
end
68+
end
69+
70+
defp check_port(nil, want_port, _want_level) do
71+
{:error, :absent, "expected a CPU on port #{want_port}, but that port is absent"}
72+
end
73+
74+
defp check_port(port, want_port, want_level) do
75+
level = port["cpu_level"]
76+
77+
cond do
78+
port["type"] != @cpu ->
79+
{:error, :not_cpu,
80+
"port #{want_port} is #{type_name(port["type"])}, NOT a CPU — the dummy never " <>
81+
"finished character-select setup (GOTCHAS #57)"}
82+
83+
want_level && level && level != want_level ->
84+
{:error, :wrong_level,
85+
"port #{want_port} is a CPU but level #{level}, expected #{want_level} — Melee " <>
86+
"defaults a fresh CPU to 1, so this is the autostart race firing before the " <>
87+
"slider drag finished"}
88+
89+
true ->
90+
:ok
91+
end
92+
end
93+
end

scripts/check_replay_ports.exs

Lines changed: 17 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -62,13 +62,9 @@ print(json.dumps(out))
6262
{json, 0} = System.cmd(python, ["-c", script | paths], stderr_to_stdout: false)
6363
reports = Jason.decode!(json)
6464

65-
type_name = fn
66-
0 -> "HUMAN"
67-
1 -> "CPU"
68-
2 -> "DEMO"
69-
3 -> "empty"
70-
other -> "type=#{inspect(other)}"
71-
end
65+
alias ExPhil.Eval.PortCheck
66+
67+
type_name = &PortCheck.type_name/1
7268

7369
failures =
7470
Enum.reduce(reports, 0, fn r, acc ->
@@ -84,38 +80,20 @@ failures =
8480

8581
want_port = opts[:expect_cpu]
8682

87-
if want_port do
88-
port = Enum.find(r["ports"], &(&1["port"] == want_port))
89-
90-
cond do
91-
is_nil(port) ->
92-
Output.error(" expected a CPU on port #{want_port}, but that port is absent")
93-
acc + 1
94-
95-
port["type"] != 1 ->
96-
Output.error(
97-
" port #{want_port} is #{type_name.(port["type"])}, NOT a CPU — the dummy " <>
98-
"never finished character-select setup (GOTCHAS #57)"
99-
)
100-
101-
acc + 1
102-
103-
opts[:expect_level] && port["cpu_level"] &&
104-
port["cpu_level"] != opts[:expect_level] ->
105-
Output.error(
106-
" port #{want_port} is a CPU but level #{port["cpu_level"]}, " <>
107-
"expected #{opts[:expect_level]} — Melee defaults a fresh CPU to 1, " <>
108-
"so this is the autostart race firing before the slider drag finished"
109-
)
110-
111-
acc + 1
112-
113-
true ->
114-
Output.success(" port #{want_port} is a CPU as requested")
115-
acc
116-
end
117-
else
118-
acc
83+
# Decision lives in ExPhil.Eval.PortCheck so it is unit-tested
84+
# (test/exphil/eval/port_check_test.exs) — this gate had none until
85+
# 2026-08-03 despite guarding every CPU-dummy eval block.
86+
case PortCheck.verify(r["ports"],
87+
expect_cpu: want_port,
88+
expect_level: opts[:expect_level]
89+
) do
90+
:ok ->
91+
if want_port, do: Output.success(" port #{want_port} is a CPU as requested")
92+
acc
93+
94+
{:error, _reason, message} ->
95+
Output.error(" " <> message)
96+
acc + 1
11997
end
12098

12199
err ->

test/exphil/data/action_frame_convention_test.exs

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,19 @@
11
defmodule ExPhil.Data.ActionFrameConventionTest do
2+
@moduledoc """
3+
⚠️ **DEPRECATED SUBJECT — these tests pin the CONTENTS of a table that
4+
GOTCHAS #81 (SETTLED 2026-07-26) declares INVALID.**
5+
6+
The live-vs-parsed `action_frame` delta varies *per occurrence* — same
7+
action, same run, same port, sometimes 0 and sometimes 1 — so no
8+
per-action key can be correct; the table is a majority vote that is wrong
9+
for the minority. These tests remain only as anti-drift pins on the
10+
measurement artifact (if the table changes, that should be deliberate).
11+
12+
**They do NOT license using the table.** `af_convention: :live` must stay
13+
off; that is guarded separately by
14+
`test/exphil/harness/af_convention_default_test.exs`, because everything
15+
here passes whether the feature is on or off.
16+
"""
217
use ExUnit.Case, async: true
318

419
alias ExPhil.Data.ActionFrameConvention, as: AFC
Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
defmodule ExPhil.Eval.PortCheckTest do
2+
@moduledoc """
3+
Pins the dummy-port decision (GOTCHAS #57 / #57b) — the check that gates
4+
every CPU-dummy eval block and had no tests until 2026-08-03, despite the
5+
bug it guards having silently invalidated the entire combo-drill era of
6+
recordings (five of six on 2026-07-26 came up HUMAN).
7+
"""
8+
use ExUnit.Case, async: true
9+
10+
alias ExPhil.Eval.PortCheck
11+
12+
defp cpu(port, level), do: %{"port" => port, "type" => 1, "cpu_level" => level}
13+
defp human(port), do: %{"port" => port, "type" => 0, "cpu_level" => nil}
14+
15+
describe "verify/2" do
16+
test "passes when the requested port is a CPU at the requested level" do
17+
assert PortCheck.verify([human(1), cpu(2, 9)], expect_cpu: 2, expect_level: 9) == :ok
18+
end
19+
20+
test "catches THE bug: a HUMAN port where a CPU was requested" do
21+
assert {:error, :not_cpu, msg} =
22+
PortCheck.verify([human(1), human(2)], expect_cpu: 2)
23+
24+
assert msg =~ "NOT a CPU"
25+
assert msg =~ "#57", "the message must name the gotcha so the reader finds the cause"
26+
end
27+
28+
test "catches the autostart race: CPU present but at Melee's default level 1" do
29+
assert {:error, :wrong_level, msg} =
30+
PortCheck.verify([human(1), cpu(2, 1)], expect_cpu: 2, expect_level: 9)
31+
32+
assert msg =~ "level 1"
33+
assert msg =~ "expected 9"
34+
end
35+
36+
test "catches an absent port" do
37+
assert {:error, :absent, _} = PortCheck.verify([human(1)], expect_cpu: 2)
38+
end
39+
40+
test "tolerates a build that does not report cpu_level" do
41+
# Level is only checked when the replay reports one — some builds
42+
# return nil, and failing there would reject good recordings.
43+
assert PortCheck.verify([human(1), cpu(2, nil)], expect_cpu: 2, expect_level: 9) == :ok
44+
end
45+
46+
test "no expectation means no check (the default eval path)" do
47+
assert PortCheck.verify([human(1), human(2)]) == :ok
48+
end
49+
end
50+
51+
describe "type_name/1" do
52+
test "names every Slippi port type" do
53+
assert PortCheck.type_name(0) == "HUMAN"
54+
assert PortCheck.type_name(1) == "CPU"
55+
assert PortCheck.type_name(2) == "DEMO"
56+
assert PortCheck.type_name(3) == "empty"
57+
assert PortCheck.type_name(nil) =~ "type="
58+
end
59+
end
60+
end
61+
62+
defmodule ExPhil.Test.AnalogReleaseEdgeTest do
63+
@moduledoc """
64+
GOTCHAS #66: an equivalence check is only as strong as its tolerance
65+
classes. The scenario drift check treated the shield family 178-182 as
66+
equivalent, so a broken analog RELEASE (EXI inputs latch neutral) scored
67+
"15/15 exact" while P2 rode shield to break and dizzy every stock.
68+
69+
`ReplicationCheck` had the same blindness in a different place — it
70+
compared the digital l/r bits and ignored the analog shoulder axis
71+
entirely. This pins that a release EDGE is now visible to `:exact`.
72+
"""
73+
use ExUnit.Case, async: true
74+
75+
alias ExPhil.Bridge.ControllerState
76+
alias ExPhil.Test.ReplicationCheck
77+
78+
defp ctrl(shoulder) do
79+
%ControllerState{
80+
main_stick: %{x: 0.5, y: 0.5},
81+
c_stick: %{x: 0.5, y: 0.5},
82+
l_shoulder: shoulder,
83+
r_shoulder: 0.0,
84+
button_a: false,
85+
button_b: false,
86+
button_x: false,
87+
button_y: false,
88+
button_z: false,
89+
button_l: false,
90+
button_r: false,
91+
button_d_up: false
92+
}
93+
end
94+
95+
test "a latched analog release is NOT scored as exact" do
96+
# expected: press then RELEASE. actual: press then LATCH (never releases)
97+
expected = [ctrl(1.0), ctrl(1.0), ctrl(0.0)]
98+
latched = [ctrl(1.0), ctrl(1.0), ctrl(1.0)]
99+
100+
assert {:error, diag} = ReplicationCheck.check(expected, latched, strictness: :exact)
101+
102+
refute diag.pass,
103+
"a latched analog release must fail :exact — this is the #66 blindness that " <>
104+
"turned 'release is broken' into 15/15 exact"
105+
end
106+
107+
test "an identical analog stream still passes" do
108+
stream = [ctrl(1.0), ctrl(1.0), ctrl(0.0)]
109+
assert {:ok, diag} = ReplicationCheck.check(stream, stream, strictness: :exact)
110+
assert diag.pass
111+
end
112+
end

test/exphil/eval/state_stream_diff_test.exs

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,15 @@
11
defmodule ExPhil.Eval.StateStreamDiffTest do
2+
@moduledoc """
3+
⚠️ The `action_frame` convention portion of these tests pins a table
4+
GOTCHAS #81 (SETTLED 2026-07-26) declares **INVALID** — the live-vs-parsed
5+
delta varies per occurrence, so no per-action key is correct. Kept as
6+
anti-drift pins on the measurement artifact only; they do not license
7+
using the table, and they pass whether `af_convention: :live` is on or
8+
off. The actual guard is
9+
`test/exphil/harness/af_convention_default_test.exs`.
10+
11+
The non-convention tests here (stream diffing itself) remain valid.
12+
"""
213
use ExUnit.Case, async: true
314

415
alias ExPhil.Eval.StateStreamDiff
Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
defmodule ExPhil.Harness.AfConventionDefaultTest do
2+
@moduledoc """
3+
Pins `af_convention: :parsed` as the default, permanently.
4+
5+
GOTCHAS #81 SETTLED (2026-07-26): the per-action delta table underlying
6+
`af_convention: :live` is **invalid**. The dual-port experiment showed the
7+
live-vs-parsed `action_frame` delta varies *per occurrence* — the same
8+
action, same run, same port, sometimes 0 and sometimes 1 — so no key
9+
(action, character, or run) makes the table right. It is a majority vote
10+
that is wrong for the minority.
11+
12+
Why this test exists: `test/exphil/data/action_frame_convention_test.exs`
13+
and `test/exphil/eval/state_stream_diff_test.exs` still pin that table's
14+
CONTENTS, so they pass whether or not the feature is enabled. A
15+
well-meaning future change flipping the default to `:live` would pass the
16+
entire suite while silently corrupting live embeddings for the minority
17+
of frames. This is the guard that catches it.
18+
19+
If you are here because this test failed: read GOTCHAS #81's SETTLED
20+
section before changing it. Re-enabling the feature requires conditioning
21+
on entry context (not action id), not a default flip.
22+
"""
23+
use ExUnit.Case, async: true
24+
25+
alias ExPhil.Agents.Agent
26+
27+
describe "af_convention default (GOTCHAS #81 SETTLED)" do
28+
test "an Agent started without the option uses :parsed" do
29+
{:ok, agent} = Agent.start_link([])
30+
31+
on_exit(fn -> if Process.alive?(agent), do: GenServer.stop(agent) end)
32+
33+
assert :sys.get_state(agent).af_convention == :parsed,
34+
"af_convention must default to :parsed — the :live delta table is INVALID " <>
35+
"(GOTCHAS #81 SETTLED: the delta varies per OCCURRENCE, so no per-action " <>
36+
"key can be correct). Re-enabling needs entry-context conditioning, not a " <>
37+
"default flip."
38+
end
39+
40+
test "the --live-af CLI flag still defaults to false" do
41+
opts = ExPhil.CLI.parse_args([], flags: [:dolphin])
42+
43+
refute opts[:live_af],
44+
"--live-af must default to false; it enables the invalid per-action delta table."
45+
end
46+
end
47+
end

test/support/replication_check.ex

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -188,13 +188,26 @@ defmodule ExPhil.Test.ReplicationCheck do
188188
bucket(e.c_stick.y, buckets) == bucket(a.c_stick.y, buckets)
189189
end
190190

191+
# GOTCHAS #66's lesson, applied: "an equivalence check is only as strong as
192+
# its tolerance classes." This compared only the DIGITAL l/r bits and was
193+
# blind to the analog shoulder axis entirely — the same blindness that let
194+
# a broken analog RELEASE score 15/15 exact under the scenario drift
195+
# check. `:exact` now means exact, analog included (quantized to 1/64 so
196+
# float noise from the decode path is not mistaken for a real difference).
191197
defp buttons_equal?(e, a) do
192198
e.button_a == a.button_a and e.button_b == a.button_b and
193199
e.button_x == a.button_x and e.button_y == a.button_y and
194200
e.button_z == a.button_z and e.button_l == a.button_l and
195-
e.button_r == a.button_r and e.button_d_up == a.button_d_up
201+
e.button_r == a.button_r and e.button_d_up == a.button_d_up and
202+
analog_equal?(e.l_shoulder, a.l_shoulder) and
203+
analog_equal?(e.r_shoulder, a.r_shoulder)
196204
end
197205

206+
defp analog_equal?(e, a) when is_number(e) and is_number(a),
207+
do: round(e * 64) == round(a * 64)
208+
209+
defp analog_equal?(e, a), do: e == a
210+
198211
# Map a 0..1 stick value to a discrete bucket index (matches the uniform
199212
# discretization used for training targets in ExPhil.Embeddings.Controller).
200213
defp bucket(v, buckets) when is_number(v), do: round(v * buckets)

0 commit comments

Comments
 (0)