results-a10-adal.tsv

commit	val_bpb	memory_gb	status	description
dc71429	1.322602	11.4	keep	baseline
d4c9600	0.000000	0.0	crash	increase DEPTH 8->12 (OOM, only 22GB VRAM)
(reset)	1.547798	17.4	discard	increase DEPTH 8->10 (too slow, only 77 steps)
c56b756	1.200468	11.4	keep	halve TOTAL_BATCH_SIZE 2^19->2^18 for 2x more steps
685abe8	1.207553	11.4	discard	TOTAL_BATCH_SIZE 2^17 (worse, too small batch)
03e2a26	1.198864	11.4	keep	increase MATRIX_LR 0.04->0.05
947a98f	1.196022	11.4	keep	increase MATRIX_LR 0.05->0.06
0da12d0	1.194746	11.4	keep	increase MATRIX_LR 0.06->0.07
2c947c3	1.198157	11.4	discard	MATRIX_LR 0.08 (worse, too high)
3ed8fab	1.197774	11.4	discard	WARMDOWN_RATIO 0.3 (slightly worse)
223b689	0.000000	0.0	crash	DEVICE_BATCH_SIZE 64 (OOM)
(reset)	1.200410	11.4	discard	WINDOW_PATTERN L (full attention, slightly worse)
e500c64	1.197005	11.4	discard	EMBEDDING_LR 0.9 (slightly worse)
dcc0ebd	1.198217	11.4	discard	WEIGHT_DECAY 0.1 (slightly worse)
(reset)	1.236701	14.1	discard	ASPECT_RATIO 80 (wider model, slower, worse)
f6cb08b	1.170294	8.8	keep	decrease ASPECT_RATIO 64->48 (smaller/faster, more steps)
b677b78	1.164274	6.1	keep	decrease ASPECT_RATIO 48->32 (even smaller/faster)
cdb90f9	1.164682	6.1	discard	ASPECT_RATIO 24 (same as 32 after rounding, no change)
beb7e02	1.160315	6.1	keep	TOTAL_BATCH_SIZE 2^17 with ASPECT_RATIO=32
eacd245	1.138181	6.1	keep	TOTAL_BATCH_SIZE 2^16 (even more steps)
d500bc1	0.000000	0.0	crash	TOTAL_BATCH_SIZE 2^15 assertion fail (batch too small)
b795b81	1.157800	3.1	discard	TOTAL_BATCH_SIZE 2^15 DEVICE_BATCH_SIZE=16 (worse)
(reset)	1.143802	11.1	discard	DEPTH=16 ASPECT_RATIO=16 (deeper, slightly worse)
794815a	1.141449	6.1	discard	HEAD_DIM=64 (4 heads, slightly worse)
d2d61ec	1.262778	2.3	discard	DEPTH=4 (too shallow, much worse)
aab33da	1.145096	5.1	discard	MLP width 2x (faster but worse)
7662c97	1.142592	4.8	discard	DEPTH=6 (worse than 8)
91bbf2f	1.137106	6.1	keep	ADAM_BETAS beta1 0.8->0.9
0cea158	1.140579	6.1	discard	ADAM_BETAS beta1 0.95 (worse)
170ffeb	1.137066	6.1	keep	UNEMBEDDING_LR 0.004->0.008
086a505	1.143570	6.1	discard	MATRIX_LR 0.09 (worse)
7190821	1.139942	6.1	discard	WARMUP_RATIO 0.05 (worse)
014808b	1.139154	6.1	discard	FINAL_LR_FRAC 0.1 (worse)
575042d	1.129891	6.1	keep	WEIGHT_DECAY 0.2->0.0
ae5f444	1.128417	6.1	keep	MATRIX_LR 0.07->0.08 with WEIGHT_DECAY=0
7999339	1.129178	6.1	discard	MATRIX_LR 0.09 (slightly worse)
ef43be9	1.127963	6.1	keep	SCALAR_LR 0.5->0.1
c03dfac	1.137747	6.1	discard	SCALAR_LR 0.02 (worse)
362ff9f	1.129383	6.1	discard	EMBEDDING_LR 1.2 (worse)
f34ec31	1.128234	6.1	discard	ADAM_BETAS beta2 0.99 (slightly worse)
f3f3c89	1.178494	6.1	discard	HEAD_DIM=32 8 heads (much worse)
64b7521	1.131081	6.1	discard	muon momentum ramp 300->100 steps (worse)
aab7701	1.128210	6.1	discard	WARMDOWN_RATIO 0.4 (slightly worse)
13a2001	1.139265	5.6	discard	MQA n_kv_head=1 (worse)
961a801	1.124832	8.7	keep	ASPECT_RATIO 32->40 (model_dim=384)
d0f9e2f	1.133088	4.5	discard	TOTAL_BATCH_SIZE 2^15 with ASPECT_RATIO=40 (worse)
13541fb	1.129721	8.7	discard	MATRIX_LR 0.10 with ASPECT_RATIO=40 (worse)
8aa3e2a	1.187968	8.8	discard	TOTAL_BATCH_SIZE 2^17 with ASPECT_RATIO=40 (worse)
c96ec79	1.126144	8.7	discard	muon beta2 0.99 (slightly worse)
c47d162	1.137385	8.7	discard	muon ns_steps 3 (worse, less orthogonalization)
fa6ade0	1.208546	8.7	discard	SwiGLU activation (worse than relu^2)
16d9b37	1.134873	11.3	discard	ASPECT_RATIO=64 with tuned hyperparams (worse)
(reset)	1.147922	13.8	discard	DEPTH=10 ASPECT_RATIO=40 (worse)
d83fbd0	1.122811	8.7	keep	muon max momentum 0.95->0.90
9bafbdc	1.122087	8.7	keep	muon momentum fixed 0.85 (no ramp)
a64c960	1.122145	8.7	discard	muon momentum 0.80 (tied, keep simpler)
0852859	1.126239	8.7	discard	cosine warmdown (worse than linear)
4801b31	1.123065	8.7	discard	EMBEDDING_LR 0.8 (slightly worse)
d861202	1.122408	8.7	discard	WARMDOWN_RATIO 0.6 (slightly worse)
ed1a4fb	1.123739	8.7	discard	WINDOW_PATTERN SSL (slightly worse)
1e6b8ab	1.125960	8.7	discard	HEAD_DIM=96 4 heads at 384-dim (slightly worse)
a6f930c	1.123019	8.7	discard	UNEMBEDDING_LR 0.004 (slightly worse than 0.008)
56aed32	1.122918	8.7	discard	grad clip 1.0 (essentially tied, keep simple)
7347cc7	1.144506	8.4	discard	disable value embeddings (worse)
986c224	1.126795	8.7	discard	softcap 15->30 (worse)
805c0d4	1.129114	8.7	discard	remove softcap (worse)
0505114	1.119339	8.7	keep	short window seq_len//4 (faster S-layer attention)
b9d13ad	1.116914	8.7	keep	short window seq_len//8
f010559	1.116308	8.7	keep	short window seq_len//16
2ad4471	1.115415	8.7	keep	short window seq_len//32
6ca69f9	1.115489	8.7	discard	short window seq_len//64 (slightly worse)
1ea3f36	1.122098	6.1	discard	ASPECT_RATIO=32 with short window//32 (worse)
edac549	1.120300	4.5	discard	TOTAL_BATCH_SIZE 2^15 with short window//32 (worse)
e7f3c9a	1.116121	8.7	discard	MATRIX_LR 0.09 with short window//32 (slightly worse)
68a8b6e	1.115490	8.7	discard	ASPECT_RATIO=48 with short window//32 (tied)
24fb7ac	1.118130	10.2	discard	MLP 6x expansion (worse, too slow)
b45e78c	1.115868	8.7	discard	WINDOW_PATTERN=SL (slightly worse)
712145f	1.116220	8.7	discard	WINDOW_PATTERN=SSSSL (slightly worse)
6b4473c	1.119816	8.7	discard	WINDOW_PATTERN=S (all short, slightly worse)
1f95b13	1.114161	8.7	keep	RoPE base 10000->50000
7d13534	1.113743	8.7	keep	RoPE base 50000->100000
9002b8f	1.115109	8.7	discard	RoPE base 500000 (worse)
ecd991b	1.114455	8.7	discard	RoPE base 200000 (worse than 100000)
1efcb8d	1.114991	8.7	discard	SCALAR_LR 0.05 (slightly worse)
5f3ce84	1.114974	8.7	discard	EMBEDDING_LR 0.4 (slightly worse)
fa07c0d	1.112640	8.9	keep	value embeddings on all layers
e42ba56	1.113829	8.9	discard	MATRIX_LR 0.09 with all VEs (slightly worse)
50b9b75	1.112828	8.9	discard	ASPECT_RATIO=48 with all VEs (tied)
cd6b839	1.175253	9.1	discard	TOTAL_BATCH_SIZE 2^17 with all VEs (much worse)
24b96dc	1.113743	8.9	discard	UNEMBEDDING_LR 0.016 (slightly worse)
6d84fb3	1.114202	8.9	discard	ve_gate_channels 64 (slightly worse)
6dde5a9	1.114017	8.9	discard	remove dmodel LR scaling (slightly worse)
b242e39	1.112908	8.9	discard	weight tying lm_head<->wte (tied)
efc55a7	1.118053	8.6	discard	remove x0 skip connection (worse)
b08dd56	1.111848	7.0	keep	DEPTH=6 ASPECT_RATIO=64 (384-dim, fewer layers)
791af83	1.114373	6.0	discard	DEPTH=5 ASPECT_RATIO=64 (worse)
(reset)	1.117869	10.3	discard	DEPTH=7 ASPECT_RATIO=64 (worse)
7090e0b	1.112647	7.0	discard	MATRIX_LR 0.09 with DEPTH=6 (slightly worse)
129dbd7	1.118387	3.6	discard	TOTAL_BATCH_SIZE 2^15 with DEPTH=6 (worse)
01034b5	1.113531	9.0	discard	ASPECT_RATIO=80 DEPTH=6 (slightly worse)
3061acf	1.113843	7.0	discard	WINDOW_PATTERN=SSSSL with DEPTH=6 (slightly worse)
4e40814	1.114786	7.0	discard	short_window //16 with DEPTH=6 (worse)
d6e8b53	1.112513	7.0	keep	RoPE base 150000 with DEPTH=6
b842c59	1.112667	7.0	discard	RoPE base 130000 (slightly worse than 150000)
78f17f6	1.111890	7.0	discard	RoPE base 170000 (tied with best)
bf02ec4	1.112763	7.0	discard	EMBEDDING_LR 0.8 with DEPTH=6 (slightly worse)
a4f5c6b	1.111989	7.0	discard	ADAM_BETAS beta1 0.85 (tied)
dae060a	1.113386	9.0	discard	ASPECT_RATIO=72 DEPTH=6 (model_dim=512, worse)
b701bd6	1.111539	7.0	keep	short_window //64 with DEPTH=6
d9902f0	1.111022	7.0	keep	short_window //128 with DEPTH=6
5d68495	1.113003	7.0	discard	short_window //256 (worse than //128)
de9ad8c	1.112782	7.0	discard	short_window //192 (worse than //128)
0268c3b	1.110752	7.0	keep	MATRIX_LR 0.07 with DEPTH=6 short_window//128
9f4f4c9	1.110885	7.0	discard	MATRIX_LR 0.06 (slightly worse)
b2ccbff	1.110690	7.0	keep	WARMDOWN_RATIO 0.6
a249fd1	1.112049	7.0	discard	WARMDOWN_RATIO 0.7 (worse)
49f8a91	1.110170	7.0	keep	FINAL_LR_FRAC 0.05
0fea32b	1.110426	7.0	discard	FINAL_LR_FRAC 0.1 (slightly worse)
6cf16a7	1.110435	7.0	discard	FINAL_LR_FRAC 0.03 (slightly worse)
51aa50c	1.109572	7.0	keep	SCALAR_LR 0.1->0.2
d830cb6	1.109953	7.0	discard	SCALAR_LR 0.3 (slightly worse)
6f1336e	1.109931	7.0	discard	MATRIX_LR 0.075 (slightly worse)
f318beb	1.109714	7.0	discard	EMBEDDING_LR 0.5 (slightly worse)
1a707cd	1.110549	7.0	discard	muon momentum 0.80 (worse)
679b8c4	1.110968	7.0	discard	SCALAR_LR 0.5 (worse)
5b816cf	1.109745	7.0	discard	ADAM_BETAS beta2 0.98 (slightly worse)
be6eb8d	1.109650	7.0	discard	FINAL_LR_FRAC 0.07 (slightly worse)
aed15f4	0.000000	0.0	crash	DEVICE_BATCH_SIZE 64 assertion fail
f96e41f	1.109407	7.0	keep	muon beta2 0.95->0.90
1a4ace4	1.109519	7.0	discard	muon beta2 0.85 (slightly worse)
2c00c1a	1.109668	7.0	discard	muon beta2 0.92 (slightly worse)
2a36e3a	1.109559	7.0	discard	muon ns_steps 7 (slightly worse)
231a4c8	1.111072	7.0	discard	UNEMBEDDING_LR 0.012 (worse)
6fcb7c2	1.109688	7.0	discard	EMBEDDING_LR 0.7 (slightly worse)
a4de92a	1.108127	7.0	keep	WEIGHT_DECAY 0.0->0.05
373e8b6	1.111510	7.0	discard	WEIGHT_DECAY 0.1 (worse)
8416c8c	1.107558	7.0	keep	WEIGHT_DECAY 0.05->0.03
071bac9	1.107679	7.0	discard	WEIGHT_DECAY 0.02 (slightly worse)
dc31c12	1.107703	7.0	discard	WEIGHT_DECAY 0.04 (slightly worse)
5d02c35	1.107688	7.0	discard	MATRIX_LR 0.065 (slightly worse)
70563a3	1.107745	7.0	discard	SCALAR_LR 0.25 (slightly worse)
d77af38	1.111391	7.0	discard	WARMUP_RATIO 0.05 (worse)
01a315d	1.107319	7.0	keep	ADAM_BETAS beta1 0.9->0.88
02416a8	1.107110	7.0	keep	ADAM_BETAS beta1 0.88->0.86
c9a2993	1.106856	7.0	keep	ADAM_BETAS beta1 0.86->0.84
a9d9e98	1.106940	7.0	discard	ADAM_BETAS beta1 0.82 (slightly worse)
cce4ad1	1.107017	7.0	discard	ADAM_BETAS beta1 0.83 (slightly worse)
120774a	1.106768	7.0	keep	ADAM_BETAS beta2 0.95->0.96
07e341c	1.106726	7.0	keep	ADAM_BETAS beta2 0.96->0.97
0a2b621	1.106788	7.0	discard	ADAM_BETAS beta2 0.975 (slightly worse)
6d7d182	1.107261	7.0	discard	MATRIX_LR 0.08 with new betas (worse)
d3339d9	1.106497	7.0	keep	EMBEDDING_LR 0.6->0.65
049f743	1.106844	7.0	discard	EMBEDDING_LR 0.7 (slightly worse)
3cf9e16	1.106598	7.0	discard	SCALAR_LR 0.22 (slightly worse)
c84e287	1.107464	7.0	discard	UNEMBEDDING_LR 0.006 (worse)
56ab1c2	1.108162	7.0	discard	UNEMBEDDING_LR 0.01 (worse)
d3c1105	1.106936	7.0	discard	EMBEDDING_LR 0.62 (slightly worse)
5a6dfa7	1.106596	7.0	discard	WEIGHT_DECAY 0.025 (slightly worse)
d60a9c7	1.106575	7.0	discard	WARMDOWN_RATIO 0.65 (slightly worse)
e79ce7e	1.106763	7.0	discard	FINAL_LR_FRAC 0.06 (slightly worse)
202216a	1.106786	7.0	discard	ADAM_BETAS beta2 0.96 with EMBEDDING_LR=0.65 (slightly worse)
4c7d2a7	1.106227	7.0	keep	short_window //96 (between //64 and //128)
fc4df01	1.105598	7.0	keep	short_window //80
7580316	1.105466	7.0	keep	short_window //70
534802f	1.105605	7.0	discard	short_window //60 (slightly worse)
8842fbb	1.105812	7.0	discard	short_window //75 (slightly worse)
9bd57db	1.105514	7.0	discard	SCALAR_LR 0.18 (slightly worse)
246b85c	1.105318	7.0	keep	MATRIX_LR 0.07->0.072
0cea3ee	1.105469	7.0	discard	MATRIX_LR 0.074 (slightly worse)
d9ec329	1.105335	7.0	discard	MATRIX_LR 0.071 (slightly worse)
f2c65de	1.105342	7.0	discard	WEIGHT_DECAY 0.035 (slightly worse)
66467d2	1.105356	7.0	discard	RoPE base 160000 (slightly worse)
d20b343	1.105753	7.0	discard	RoPE base 140000 (slightly worse)
4d096b3	1.105278	7.0	keep	EMBEDDING_LR 0.65->0.67
c703d69	1.105465	7.0	discard	EMBEDDING_LR 0.69 (slightly worse)
09a529b	1.105331	7.0	discard	muon momentum 0.87 (slightly worse)
0d2fc84	1.105433	7.0	discard	SCALAR_LR 0.21 (slightly worse)
dae5a05	1.105532	7.0	discard	ADAM_BETAS beta1 0.845 (slightly worse)
35df374	1.105252	7.0	keep	ADAM_BETAS beta1 0.84->0.835
14397bb	1.105215	7.0	keep	ADAM_BETAS beta1 0.835->0.83
b59959d	1.105064	7.0	keep	ADAM_BETAS beta1 0.83->0.825
12a56af	1.105109	7.0	discard	ADAM_BETAS beta1 0.82 (slightly worse)
ab74ae9	1.105168	7.0	discard	ADAM_BETAS beta2 0.968 (slightly worse)
baf8f6c	1.105102	7.0	discard	ADAM_BETAS beta2 0.972 (slightly worse)
f24e5aa	1.105185	7.0	discard	WEIGHT_DECAY 0.032 (slightly worse)
f183178	1.105153	7.0	discard	EMBEDDING_LR 0.66 (slightly worse)
010392d	1.105177	7.0	discard	MATRIX_LR 0.073 (slightly worse)
dcd3e8e	1.105268	7.0	discard	EMBEDDING_LR 0.68 (slightly worse)
ab12afe	1.105240	7.0	discard	WARMDOWN_RATIO 0.58 (slightly worse)
8ee875b	1.105225	7.0	discard	SCALAR_LR 0.19 (slightly worse)
3a6a55e	1.105273	7.0	discard	RoPE base 155000 (slightly worse)
1ad1497	1.105019	7.0	keep	ADAM_BETAS beta1 0.825->0.826
51571f9	1.105197	7.0	discard	ADAM_BETAS beta1 0.827 (slightly worse)
311e72a	1.105164	7.0	discard	ADAM_BETAS beta1 0.8255 (slightly worse)
25b3e20	1.105131	7.0	discard	ADAM_BETAS beta2 0.971 (slightly worse)
fef3547	1.105177	7.0	discard	ADAM_BETAS beta2 0.969 (slightly worse)
e51bc60	1.105200	7.0	discard	EMBEDDING_LR 0.675 (slightly worse)
2e06781	1.105211	7.0	discard	MATRIX_LR 0.0715 (slightly worse)
fc0d5a2	1.105192	7.0	discard	WEIGHT_DECAY 0.028 (slightly worse)
e426085	1.105643	7.0	discard	UNEMBEDDING_LR 0.009 (worse)
02848b4	1.105291	7.0	discard	WEIGHT_DECAY 0.031 (slightly worse)
c86739d	1.105241	7.0	discard	EMBEDDING_LR 0.671 (slightly worse)
c75240e	1.105204	7.0	discard	ADAM_BETAS beta2 0.966 (slightly worse)
fb6ac88	1.105253	7.0	discard	ADAM_BETAS beta2 0.9705 (slightly worse)
2d10658	1.105104	7.0	discard	ADAM_BETAS beta1 0.824 (slightly worse)
f54cb9e	1.105688	7.0	discard	short_window //65 (worse than //70)
1bd2c26	1.105380	7.0	discard	short_window //68 (worse than //70)
f8a2b2b	1.105411	7.0	discard	WARMDOWN_RATIO 0.55 (slightly worse)
f79baba	1.105447	7.0	discard	WEIGHT_DECAY 0.033 (slightly worse)
4805bfc	1.105144	7.0	discard	muon beta2 0.88 (slightly worse)
9cff570	1.105336	7.0	discard	muon beta2 0.91 (slightly worse)
8e705d4	1.105288	7.0	discard	WEIGHT_DECAY 0.029 (slightly worse)
f09afe6	1.105390	7.0	discard	MUON_MOMENTUM 0.84 (slightly worse than 0.85)
809e7a8	1.113441	6.3	discard	DEPTH=8 ASPECT_RATIO=32 with best hyperparams (worse)
88dd018	1.104955	7.0	keep	MUON_MOMENTUM=0.86
c609da9	1.105190	7.0	discard	MUON_MOMENTUM=0.87 (worse)
d738053	1.105069	7.0	discard	MUON_MOMENTUM=0.855 (slightly worse)
770a347	1.105111	7.0	discard	MUON_MOMENTUM=0.858 (slightly worse)
d6bf86c	1.105210	7.0	discard	MUON_MOMENTUM=0.862 (slightly worse)
e04a5cf	1.105002	7.0	discard	MUON_MOMENTUM=0.861 (tied)
d52123b	1.105279	7.0	discard	MUON_MOMENTUM=0.8605 (slightly worse)
fd436e3	1.105318	7.0	discard	MUON_MOMENTUM=0.8595 (slightly worse)
78f0569	1.108437	6.4	discard	MLP 3x (worse, less capacity)
651c5a3	1.105200	7.0	discard	MUON_BETA2=0.895 (slightly worse than 0.90)
93b9f02	1.105433	7.0	discard	ADAM_BETAS beta2 0.965 (worse)
0d52a25	1.105188	7.0	discard	ADAM_BETAS beta2 0.975 (slightly worse)
b3e7883	1.105183	7.0	discard	MATRIX_LR 0.073 (slightly worse)
fdee18e	1.104959	7.0	keep	SCALAR_LR 0.2->0.195
47a3842	1.105077	7.0	discard	SCALAR_LR 0.19 (slightly worse)
654025a	1.105055	7.0	discard	SCALAR_LR 0.197 (slightly worse)
7aaa670	1.105276	7.0	discard	SCALAR_LR 0.196 (slightly worse)
a4b502f	1.105029	7.0	discard	SCALAR_LR 0.194 (slightly worse)
c320f75	1.105303	7.0	discard	SCALAR_LR 0.193 (slightly worse)
a361402	1.104907	7.0	keep	SCALAR_LR 0.198
817bc6b	1.105266	7.0	discard	SCALAR_LR 0.1985 (slightly worse)
e7cab83	1.105430	7.0	discard	SCALAR_LR 0.199 (slightly worse)
4714d44	1.105305	7.0	discard	SCALAR_LR 0.1975 (slightly worse)
4e4f70f	1.105309	7.0	discard	SCALAR_LR 0.1982 (slightly worse)
c93cb04	1.105118	7.0	discard	SCALAR_LR 0.1981 (slightly worse)
52e2443	1.105158	7.0	discard	cleanup MUON_MOMENTUM/simplify (slightly worse)
3c50b73	1.105121	7.0	discard	MUON_BETA2=0.893 (slightly worse)
8deda11	1.105084	7.0	discard	simplify get_muon_momentum MUON_MOMENTUM=0.86 (slightly worse)
91979a1	1.105057	7.0	discard	simplify get_muon_momentum (slightly worse)
d830dd7	1.105029	7.0	discard	MUON_MOMENTUM=0.86 MUON_BETA2=0.896 (slightly worse)
341b033	1.104944	7.0	keep	MUON_BETA2=0.898
87ba7ca	1.105308	7.0	discard	MUON_BETA2=0.899 (worse)
316b043	1.105002	7.0	discard	MUON_BETA2=0.8975 (slightly worse)
e88568a	1.105018	7.0	discard	MUON_BETA2=0.8978 (slightly worse)
008f497	1.105005	7.0	discard	MUON_BETA2=0.8976 (slightly worse)
70431ac	1.105054	7.0	discard	MUON_BETA2=0.8979 (slightly worse)
2491d39	1.105070	7.0	discard	MUON_BETA2=0.897 (slightly worse)
2a09728	1.104876	7.0	keep	MUON_BETA2=0.8983
92dda00	1.105033	7.0	discard	MUON_BETA2=0.8986 (slightly worse)
7aeff95	1.105062	7.0	discard	MUON_BETA2=0.8984 (slightly worse)
8e50b19	1.105126	7.0	discard	MUON_BETA2=0.8981 (slightly worse)
48b3c0d	1.105118	7.0	discard	MUON_BETA2=0.8985 (slightly worse)
c96f52d	1.104900	7.0	keep	MUON_BETA2=0.8982
3b38506	1.105112	7.0	discard	MUON_BETA2=0.8981 (slightly worse)
c003a69	1.105203	7.0	discard	MUON_BETA2=0.8983 (slightly worse)
cc4dd7d	1.105210	7.0	discard	MUON_BETA2=0.8980 (slightly worse)
a103f86	1.105310	7.0	discard	MUON_BETA2=0.8982 MUON_MOMENTUM clean (slightly worse)
b9b834f	1.104989	7.0	discard	MUON_BETA2=0.8975 (slightly worse)
bf404ce	1.105102	7.0	discard	MUON_BETA2=0.8978 (slightly worse)
6b94c56	1.105229	7.0	discard	MUON_BETA2=0.8984 (slightly worse)
d1b9aad	1.104949	7.0	keep	MUON_MOMENTUM=0.86 MUON_BETA2=0.8982 clean hyperparams
4f0f234	1.105059	7.0	discard	MUON_BETA2=0.8983 (slightly worse)
dfc31f8	1.105087	7.0	discard	MUON_BETA2=0.8981 (slightly worse)
823afda	1.104907	7.0	discard	MUON_BETA2=0.898 (slightly worse)
7a269ed	1.105060	7.0	discard	MUON_NS_STEPS=6 (slightly worse)
5f170f2	1.105239	7.0	discard	MUON_NS_STEPS=4 (worse)
ff022ec	1.105148	7.0	discard	MUON_NS_STEPS=5 param add (slightly worse)
1bb9adb	1.104984	7.0	keep	add MUON_NS_STEPS=5 MUON_MOMENTUM MUON_BETA2 clean hyperparams
21d3199	1.105240	7.0	discard	MATRIX_LR 0.0725 (slightly worse)
17106ca	1.105153	7.0	discard	MATRIX_LR 0.0718 (slightly worse)
635a8f6	1.105210	7.0	discard	MATRIX_LR 0.0716 (slightly worse)
17c08d5	1.105320	7.0	discard	MATRIX_LR 0.0712 (slightly worse)
fb645bb	1.105035	7.0	discard	MATRIX_LR 0.0722 (slightly worse)
636ffa3	1.104952	7.0	discard	MATRIX_LR 0.0721 (slightly worse)
14e0074	1.105013	7.0	discard	MATRIX_LR 0.0719 (slightly worse)
9ba4807	1.105285	7.0	discard	MATRIX_LR 0.0715 (slightly worse)
2705d6f	1.105044	7.0	discard	MATRIX_LR 0.0724 (slightly worse)
8aa52e0	1.105150	7.0	discard	MATRIX_LR 0.0726 (slightly worse)
52b3a8e	1.104941	7.0	discard	MATRIX_LR 0.0723 (slightly worse)
78cc412	1.104899	7.0	discard	MATRIX_LR 0.0720 (slightly worse)
bb8aeef	1.105345	7.0	discard	MATRIX_LR 0.0717 (slightly worse)
ba576d7	1.105311	7.0	discard	MATRIX_LR 0.0714 (slightly worse)
4d4a19d	1.105202	7.0	discard	MATRIX_LR 0.0713 (slightly worse)
b53664d	1.105129	7.0	discard	MATRIX_LR 0.0711 (slightly worse)
e5309a3	1.105005	7.0	discard	MATRIX_LR 0.0710 (slightly worse)
39e4a8e	1.105078	7.0	discard	MATRIX_LR 0.0709 (slightly worse)
4b166a4	1.105193	7.0	discard	MATRIX_LR 0.0708 (slightly worse)
a18db26	1.105339	7.0	discard	MATRIX_LR 0.0716 (slightly worse)
0f1a046	1.105310	7.0	discard	MATRIX_LR 0.0705 (slightly worse)
f1a4c89	1.104996	7.0	discard	MATRIX_LR 0.0700 (slightly worse)
8eeb91d	1.105239	7.0	discard	MATRIX_LR 0.0695 (slightly worse)
e0dd8e3	1.105161	7.0	discard	MATRIX_LR 0.0688 (slightly worse)
68a3d1e	1.105085	7.0	discard	MATRIX_LR 0.0680 (slightly worse)
2b2755e	1.105376	7.0	discard	MATRIX_LR 0.0660 (worse)
2be7227	1.105010	7.0	discard	MATRIX_LR 0.0750 (slightly worse)
1d9afc5	1.106113	7.0	discard	MATRIX_LR 0.0780 (worse)
64bed87	1.105483	7.0	discard	MATRIX_LR 0.0760 (slightly worse)
a335760	1.105334	7.0	discard	MATRIX_LR 0.0740 (slightly worse)
ede7919	1.105010	7.0	discard	MATRIX_LR 0.0730 (slightly worse)
bb47161	1.105119	7.0	discard	MATRIX_LR 0.0727 (slightly worse)
4b65cad	1.105084	7.0	discard	MATRIX_LR 0.0728 (slightly worse)
61cda64	1.105092	7.0	discard	MATRIX_LR 0.0729 (slightly worse)
2854321	1.105156	7.0	discard	MATRIX_LR 0.0720 (slightly worse)
ceeef63	1.105255	7.0	discard	MATRIX_LR 0.0718 (slightly worse)
c02363e	1.105277	7.0	discard	MATRIX_LR 0.0715 (slightly worse)
caceb28	1.105121	7.0	discard	MATRIX_LR 0.0710 (slightly worse)
308f0ef	1.105281	7.0	discard	MATRIX_LR 0.0705 (slightly worse)
c49d152	1.105299	7.0	discard	MATRIX_LR 0.0701 (slightly worse)
e7b10e6	1.105117	7.0	discard	MATRIX_LR 0.0702 (slightly worse)
12e528d	1.105123	7.0	discard	MATRIX_LR 0.0703 (slightly worse)
150ff91	1.104943	7.0	discard	MATRIX_LR 0.0704 (slightly worse)
6cff552	1.105206	7.0	discard	MATRIX_LR 0.0706 (slightly worse)
f6caa8c	1.105317	7.0	discard	MATRIX_LR 0.0707 (slightly worse)
760f24b	1.105099	7.0	discard	fix MUON_NS_STEPS missing (slightly worse)
135d897	1.104815	7.0	keep	MATRIX_LR 0.0704
ec3998a	1.105015	7.0	discard	MATRIX_LR 0.0703 (slightly worse)
247f8fb	1.105013	7.0	discard	MATRIX_LR 0.0705 (slightly worse)
5168be8	1.105214	7.0	discard	MATRIX_LR 0.0706 (slightly worse)
ce3cae2	1.105131	7.0	discard	MATRIX_LR 0.0702 (slightly worse)
76fd5f2	1.105008	7.0	discard	MATRIX_LR 0.0701 (slightly worse)
6eeb366	1.105075	7.0	discard	MATRIX_LR 0.0700 (slightly worse)
cedda92	1.105141	7.0	discard	MATRIX_LR 0.0698 (slightly worse)
39fd58a	1.105242	7.0	discard	MATRIX_LR 0.0695 (slightly worse)
9020d44	1.105372	7.0	discard	MATRIX_LR 0.0690 (slightly worse)
d97df82	1.105292	7.0	discard	MATRIX_LR 0.0680 (slightly worse)
8a92087	1.105240	7.0	discard	MATRIX_LR 0.0660 (slightly worse)
2e27b2f	1.105107	7.0	discard	MATRIX_LR 0.0640 (slightly worse)
b3ecae7	1.105313	7.0	discard	MATRIX_LR 0.0620 (slightly worse)
6a2567e	1.105501	7.0	discard	MATRIX_LR 0.0600 (worse)
ae7e3a2	1.105140	7.0	discard	MATRIX_LR 0.0704 SCALAR_LR 0.200 clean (slightly worse)