Free-Loss Jacobian Spectral Target / bench_report.json
Failed on benchmark
1{
2 "bench_version": 1,
3 "track": "tabular",
4 "model": "mlp_tiny",
5 "metric_direction": "lower is better",
6 "n_seeds": 8,
7 "baseline": {
8 "best_cfg": {
9 "lr": 0.003,
10 "epochs": 8,
11 "weight_decay": 0.0,
12 "lam": 0.0
13 },
14 "sweep": [
15 {
16 "cfg": {
17 "lr": 0.001,
18 "epochs": 8,
19 "weight_decay": 0.0,
20 "lam": 0.0
21 },
22 "mean": 188.35083389282227
23 },
24 {
25 "cfg": {
26 "lr": 0.003,
27 "epochs": 8,
28 "weight_decay": 0.0,
29 "lam": 0.0
30 },
31 "mean": 19.947585344314575
32 },
33 {
34 "cfg": {
35 "lr": 0.006,
36 "epochs": 8,
37 "weight_decay": 0.0,
38 "lam": 0.0
39 },
40 "mean": 22.523260593414307
41 }
42 ],
43 "full": {
44 "mean": 21.810271620750427,
45 "std": 6.031322024802775,
46 "per_seed": [
47 17.455322265625,
48 15.50521183013916,
49 29.37735366821289,
50 17.45245361328125,
51 33.27134323120117,
52 16.96589469909668,
53 22.887645721435547,
54 21.56694793701172
55 ],
56 "n": 8
57 }
58 },
59 "idea": {
60 "mean": 21.810271620750427,
61 "std": 6.031322024802775,
62 "per_seed": [
63 17.455322265625,
64 15.50521183013916,
65 29.37735366821289,
66 17.45245361328125,
67 33.27134323120117,
68 16.96589469909668,
69 22.887645721435547,
70 21.56694793701172
71 ],
72 "n": 8
73 },
74 "comparison": {
75 "delta_mean": 0.0,
76 "idea_wins": 0,
77 "n_pairs": 8,
78 "per_seed_diffs": [
79 0.0,
80 0.0,
81 0.0,
82 0.0,
83 0.0,
84 0.0,
85 0.0,
86 0.0
87 ],
88 "p_value": 1.0,
89 "mde": 0.0,
90 "mde_rel_pct": 0.0,
91 "verdict": "no measurable effect",
92 "system_worked": false
93 },
94 "selected_idea_cfg": {
95 "lr": 0.003,
96 "epochs": 8,
97 "weight_decay": 0.0,
98 "lam": 0.0001
99 },
100 "math_check": {
101 "tau": 0.5,
102 "target": [
103 0.6065306663513184,
104 0.5518191456794739
105 ],
106 "empirical": [
107 0.600790664279383,
108 0.5476969697311627
109 ],
110 "abs_error": [
111 0.005740002071935368,
112 0.004122175948311213
113 ],
114 "confirmed": true
115 },
116 "mechanism_signature": {
117 "source": "trained mlp_tiny Jacobian on held-out tabular inputs",
118 "predicted_moments": [
119 0.9692332148551941,
120 0.9687697291374207
121 ],
122 "observed_moments": [
123 5.242537975311279,
124 34.17074203491211
125 ],
126 "log_moment_rmse": 2.7879295349121094,
127 "confirmed": false
128 },
129 "idea_sweep": [
130 {
131 "cfg": {
132 "lr": 0.001,
133 "epochs": 8,
134 "weight_decay": 0.0,
135 "lam": 0.0001
136 },
137 "result": {
138 "mean": 187.1381435394287,
139 "std": 11.829779321921778,
140 "per_seed": [
141 200.69190979003906,
142 191.61203002929688,
143 186.99557495117188,
144 174.10382080078125,
145 177.56117248535156,
146 178.70860290527344,
147 177.95843505859375,
148 209.47360229492188
149 ],
150 "n": 8
151 }
152 },
153 {
154 "cfg": {
155 "lr": 0.001,
156 "epochs": 8,
157 "weight_decay": 0.0,
158 "lam": 0.0003
159 },
160 "result": {
161 "mean": 187.1381435394287,
162 "std": 11.829779321921778,
163 "per_seed": [
164 200.69190979003906,
165 191.61203002929688,
166 186.99557495117188,
167 174.10382080078125,
168 177.56117248535156,
169 178.70860290527344,
170 177.95843505859375,
171 209.47360229492188
172 ],
173 "n": 8
174 }
175 },
176 {
177 "cfg": {
178 "lr": 0.001,
179 "epochs": 8,
180 "weight_decay": 0.0,
181 "lam": 0.001
182 },
183 "result": {
184 "mean": 187.1381435394287,
185 "std": 11.829779321921778,
186 "per_seed": [
187 200.69190979003906,
188 191.61203002929688,
189 186.99557495117188,
190 174.10382080078125,
191 177.56117248535156,
192 178.70860290527344,
193 177.95843505859375,
194 209.47360229492188
195 ],
196 "n": 8
197 }
198 },
199 {
200 "cfg": {
201 "lr": 0.003,
202 "epochs": 8,
203 "weight_decay": 0.0,
204 "lam": 0.0001
205 },
206 "result": {
207 "mean": 21.810271620750427,
208 "std": 6.031322024802775,
209 "per_seed": [
210 17.455322265625,
211 15.50521183013916,
212 29.37735366821289,
213 17.45245361328125,
214 33.27134323120117,
215 16.96589469909668,
216 22.887645721435547,
217 21.56694793701172
218 ],
219 "n": 8
220 }
221 },
222 {
223 "cfg": {
224 "lr": 0.003,
225 "epochs": 8,
226 "weight_decay": 0.0,
227 "lam": 0.0003
228 },
229 "result": {
230 "mean": 21.810271620750427,
231 "std": 6.031322024802775,
232 "per_seed": [
233 17.455322265625,
234 15.50521183013916,
235 29.37735366821289,
236 17.45245361328125,
237 33.27134323120117,
238 16.96589469909668,
239 22.887645721435547,
240 21.56694793701172
241 ],
242 "n": 8
243 }
244 },
245 {
246 "cfg": {
247 "lr": 0.003,
248 "epochs": 8,
249 "weight_decay": 0.0,
250 "lam": 0.001
251 },
252 "result": {
253 "mean": 21.810271620750427,
254 "std": 6.031322024802775,
255 "per_seed": [
256 17.455322265625,
257 15.50521183013916,
258 29.37735366821289,
259 17.45245361328125,
260 33.27134323120117,
261 16.96589469909668,
262 22.887645721435547,
263 21.56694793701172
264 ],
265 "n": 8
266 }
267 },
268 {
269 "cfg": {
270 "lr": 0.006,
271 "epochs": 8,
272 "weight_decay": 0.0,
273 "lam": 0.0001
274 },
275 "result": {
276 "mean": 22.986173629760742,
277 "std": 2.470365935456011,
278 "per_seed": [
279 25.04693603515625,
280 23.222103118896484,
281 21.88385772705078,
282 19.94014549255371,
283 19.829004287719727,
284 21.487890243530273,
285 25.67311668395996,
286 26.80633544921875
287 ],
288 "n": 8
289 }
290 },
291 {
292 "cfg": {
293 "lr": 0.006,
294 "epochs": 8,
295 "weight_decay": 0.0,
296 "lam": 0.0003
297 },
298 "result": {
299 "mean": 22.986173629760742,
300 "std": 2.470365935456011,
301 "per_seed": [
302 25.04693603515625,
303 23.222103118896484,
304 21.88385772705078,
305 19.94014549255371,
306 19.829004287719727,
307 21.487890243530273,
308 25.67311668395996,
309 26.80633544921875
310 ],
311 "n": 8
312 }
313 },
314 {
315 "cfg": {
316 "lr": 0.006,
317 "epochs": 8,
318 "weight_decay": 0.0,
319 "lam": 0.001
320 },
321 "result": {
322 "mean": 22.986173629760742,
323 "std": 2.470365935456011,
324 "per_seed": [
325 25.04693603515625,
326 23.222103118896484,
327 21.88385772705078,
328 19.94014549255371,
329 19.829004287719727,
330 21.487890243530273,
331 25.67311668395996,
332 26.80633544921875
333 ],
334 "n": 8
335 }
336 }
337 ]
338}