Schur Interaction Monitor for Adaptive Hyperparameters / bench_report.json
Failed on benchmark
1{
2 "bench_version": 1,
3 "track": "tabular",
4 "model": "mlp_tiny",
5 "metric_direction": "lower is better",
6 "n_seeds": 8,
7 "baseline": {
8 "best_cfg": {
9 "lr": 0.006,
10 "weight_decay": 0.0001
11 },
12 "sweep": [
13 {
14 "cfg": {
15 "lr": 0.001,
16 "weight_decay": 0.0
17 },
18 "mean": 9.648519039154053
19 },
20 {
21 "cfg": {
22 "lr": 0.001,
23 "weight_decay": 0.0001
24 },
25 "mean": 9.278189897537231
26 },
27 {
28 "cfg": {
29 "lr": 0.003,
30 "weight_decay": 0.0
31 },
32 "mean": 6.7801748514175415
33 },
34 {
35 "cfg": {
36 "lr": 0.003,
37 "weight_decay": 0.0001
38 },
39 "mean": 6.735160827636719
40 },
41 {
42 "cfg": {
43 "lr": 0.006,
44 "weight_decay": 0.0
45 },
46 "mean": 6.384713053703308
47 },
48 {
49 "cfg": {
50 "lr": 0.006,
51 "weight_decay": 0.0001
52 },
53 "mean": 5.684175610542297
54 }
55 ],
56 "full": {
57 "mean": 6.078357219696045,
58 "std": 0.6427959389730813,
59 "per_seed": [
60 5.252819061279297,
61 6.84149169921875,
62 5.513339042663574,
63 5.129052639007568,
64 6.107614517211914,
65 6.495948314666748,
66 6.7050628662109375,
67 6.58152961730957
68 ],
69 "n": 8
70 }
71 },
72 "idea": {
73 "mean": 6.396751523017883,
74 "std": 0.5465600136268931,
75 "per_seed": [
76 5.679222583770752,
77 6.877334117889404,
78 5.647143363952637,
79 6.208988666534424,
80 6.021716594696045,
81 6.866507053375244,
82 6.70987606048584,
83 7.163223743438721
84 ],
85 "n": 8
86 },
87 "comparison": {
88 "delta_mean": 0.3183943033218384,
89 "idea_wins": 1,
90 "n_pairs": 8,
91 "per_seed_diffs": [
92 0.4264035224914551,
93 0.0358424186706543,
94 0.1338043212890625,
95 1.0799360275268555,
96 -0.08589792251586914,
97 0.3705587387084961,
98 0.004813194274902344,
99 0.5816941261291504
100 ],
101 "p_value": 0.03725,
102 "mde": 0.32219506919545965,
103 "mde_rel_pct": 5.300693222692354,
104 "verdict": "idea worse (significant)",
105 "system_worked": false
106 },
107 "idea_sweep": [
108 {
109 "cfg": {
110 "lr": 0.001,
111 "weight_decay": 0.0001
112 },
113 "result": {
114 "mean": 9.852757096290588,
115 "std": 0.9269408122603433,
116 "per_seed": [
117 8.10088062286377,
118 10.045586585998535,
119 10.098564147949219,
120 9.331501007080078,
121 9.319607734680176,
122 9.732595443725586,
123 10.95158863067627,
124 11.241732597351074
125 ],
126 "n": 8
127 },
128 "signature_records": [
129 {
130 "seed": 0,
131 "max_R_eigenvalue": 28.970746104644462,
132 "predicted_mean_shrink": 1.422844669378787,
133 "observed_mean_shrink": 0.45361143159707396
134 },
135 {
136 "seed": 1,
137 "max_R_eigenvalue": 29.142965118671878,
138 "predicted_mean_shrink": 1.4488059888616662,
139 "observed_mean_shrink": 0.4818310892707357
140 },
141 {
142 "seed": 2,
143 "max_R_eigenvalue": 27.347786365732766,
144 "predicted_mean_shrink": 1.4408522250381386,
145 "observed_mean_shrink": 0.46370301390802937
146 },
147 {
148 "seed": 3,
149 "max_R_eigenvalue": 28.866688051615224,
150 "predicted_mean_shrink": 1.4297103460036273,
151 "observed_mean_shrink": 0.46532923014069066
152 },
153 {
154 "seed": 4,
155 "max_R_eigenvalue": 31.114504091159937,
156 "predicted_mean_shrink": 1.4415057610296693,
157 "observed_mean_shrink": 0.47638124427758577
158 },
159 {
160 "seed": 5,
161 "max_R_eigenvalue": 27.139142402157653,
162 "predicted_mean_shrink": 1.4215714981394794,
163 "observed_mean_shrink": 0.45438587932604674
164 },
165 {
166 "seed": 6,
167 "max_R_eigenvalue": 29.95125786067226,
168 "predicted_mean_shrink": 1.4126213108110166,
169 "observed_mean_shrink": 0.44818127470488955
170 },
171 {
172 "seed": 7,
173 "max_R_eigenvalue": 30.227238142474327,
174 "predicted_mean_shrink": 1.430343664080256,
175 "observed_mean_shrink": 0.45656432355707843
176 }
177 ]
178 },
179 {
180 "cfg": {
181 "lr": 0.003,
182 "weight_decay": 0.0001
183 },
184 "result": {
185 "mean": 6.776016294956207,
186 "std": 0.3423025857087886,
187 "per_seed": [
188 6.252743244171143,
189 7.0802903175354,
190 6.731771469116211,
191 6.9929890632629395,
192 6.439003944396973,
193 6.6504974365234375,
194 6.667675018310547,
195 7.393159866333008
196 ],
197 "n": 8
198 },
199 "signature_records": [
200 {
201 "seed": 0,
202 "max_R_eigenvalue": 19.40545124327589,
203 "predicted_mean_shrink": 1.5662079346101938,
204 "observed_mean_shrink": 0.5596839399958696
205 },
206 {
207 "seed": 1,
208 "max_R_eigenvalue": 18.18841998175957,
209 "predicted_mean_shrink": 1.5452502825983927,
210 "observed_mean_shrink": 0.5358931141930726
211 },
212 {
213 "seed": 2,
214 "max_R_eigenvalue": 18.48060331058384,
215 "predicted_mean_shrink": 1.596801548354272,
216 "observed_mean_shrink": 0.5895995070263204
217 },
218 {
219 "seed": 3,
220 "max_R_eigenvalue": 18.32546811217679,
221 "predicted_mean_shrink": 1.5674429841867479,
222 "observed_mean_shrink": 0.5633915474229879
223 },
224 {
225 "seed": 4,
226 "max_R_eigenvalue": 22.240504809382237,
227 "predicted_mean_shrink": 1.6088249246045303,
228 "observed_mean_shrink": 0.6079893217536149
229 },
230 {
231 "seed": 5,
232 "max_R_eigenvalue": 15.727164040816485,
233 "predicted_mean_shrink": 1.5816451385772001,
234 "observed_mean_shrink": 0.5776791584829287
235 },
236 {
237 "seed": 6,
238 "max_R_eigenvalue": 21.040818460169987,
239 "predicted_mean_shrink": 1.5729334873273007,
240 "observed_mean_shrink": 0.5678789677304293
241 },
242 {
243 "seed": 7,
244 "max_R_eigenvalue": 18.48838750187182,
245 "predicted_mean_shrink": 1.567401702259996,
246 "observed_mean_shrink": 0.5609064495439936
247 }
248 ]
249 },
250 {
251 "cfg": {
252 "lr": 0.006,
253 "weight_decay": 0.0001
254 },
255 "result": {
256 "mean": 6.396751523017883,
257 "std": 0.5465600136268931,
258 "per_seed": [
259 5.679222583770752,
260 6.877334117889404,
261 5.647143363952637,
262 6.208988666534424,
263 6.021716594696045,
264 6.866507053375244,
265 6.70987606048584,
266 7.163223743438721
267 ],
268 "n": 8
269 },
270 "signature_records": [
271 {
272 "seed": 0,
273 "max_R_eigenvalue": 21.520722249039736,
274 "predicted_mean_shrink": 1.5998354212422041,
275 "observed_mean_shrink": 0.5803018674852735
276 },
277 {
278 "seed": 1,
279 "max_R_eigenvalue": 26.758464379751487,
280 "predicted_mean_shrink": 1.568503006010591,
281 "observed_mean_shrink": 0.5511755243066208
282 },
283 {
284 "seed": 2,
285 "max_R_eigenvalue": 21.52823242514386,
286 "predicted_mean_shrink": 1.6324191170696345,
287 "observed_mean_shrink": 0.6153089706860777
288 },
289 {
290 "seed": 3,
291 "max_R_eigenvalue": 19.959858225483167,
292 "predicted_mean_shrink": 1.5989631426785875,
293 "observed_mean_shrink": 0.5828625680472822
294 },
295 {
296 "seed": 4,
297 "max_R_eigenvalue": 21.21196735513266,
298 "predicted_mean_shrink": 1.6179572148072678,
299 "observed_mean_shrink": 0.6018178223934545
300 },
301 {
302 "seed": 5,
303 "max_R_eigenvalue": 21.802852810772954,
304 "predicted_mean_shrink": 1.609761440973835,
305 "observed_mean_shrink": 0.595776442199383
306 },
307 {
308 "seed": 6,
309 "max_R_eigenvalue": 22.262030674047317,
310 "predicted_mean_shrink": 1.5987779728372298,
311 "observed_mean_shrink": 0.5819983240302995
312 },
313 {
314 "seed": 7,
315 "max_R_eigenvalue": 22.797705272860483,
316 "predicted_mean_shrink": 1.6093046685842605,
317 "observed_mean_shrink": 0.594072235302656
318 }
319 ]
320 }
321 ],
322 "math_verification": {
323 "max_abs_hessian_error": 1.0704046315979099e-08,
324 "min_R_eigenvalue": 0.052625321728098556,
325 "identity_pass": true
326 },
327 "device": "cuda",
328 "protocol_note": "Baseline sweep uses all idea learning rates and both Adam weight decays; 8 paired seeds.",
329 "mechanism_signature": {
330 "quantity": "Schur trust shrinkage measured during trained MLP updates",
331 "predicted_vs_observed": {
332 "predicted_mean_shrink": 1.6044402480254512,
333 "observed_mean_shrink": 0.5879142193063809,
334 "relative_error": 1.7290380047562637
335 },
336 "max_R_eigenvalue_mean": 22.230229174028956,
337 "confirmed": false
338 }
339}