Identity-Paired Progressive Depth / bench_report.json
Mechanism confirmed, baseline not beaten
1{
2 "bench_version": 1,
3 "track": "sequence",
4 "model": "transformer_tiny",
5 "metric_direction": "lower is better",
6 "n_seeds": 8,
7 "baseline": {
8 "best_cfg": {
9 "lr": 0.0015,
10 "weight_decay": 0.0
11 },
12 "sweep": [
13 {
14 "cfg": {
15 "lr": 0.0015,
16 "weight_decay": 0.0
17 },
18 "mean": 0.5785842016339302
19 },
20 {
21 "cfg": {
22 "lr": 0.003,
23 "weight_decay": 0.0
24 },
25 "mean": 0.6899479925632477
26 },
27 {
28 "cfg": {
29 "lr": 0.006,
30 "weight_decay": 0.0
31 },
32 "mean": 0.8105297237634659
33 }
34 ],
35 "full": {
36 "mean": 0.562101598829031,
37 "std": 0.08060035643973527,
38 "per_seed": [
39 0.6448022127151489,
40 0.4093892276287079,
41 0.5553309321403503,
42 0.7048144340515137,
43 0.5405545234680176,
44 0.5604735016822815,
45 0.5436524748802185,
46 0.5377954840660095
47 ],
48 "n": 8
49 }
50 },
51 "idea": {
52 "mean": 0.5841716527938843,
53 "std": 0.05076107251588834,
54 "per_seed": [
55 0.6083592772483826,
56 0.5841273665428162,
57 0.6231781840324402,
58 0.5197466015815735,
59 0.6223173141479492,
60 0.5627732872962952,
61 0.654932975769043,
62 0.49793821573257446
63 ],
64 "n": 8
65 },
66 "comparison": {
67 "delta_mean": 0.022070053964853287,
68 "idea_wins": 3,
69 "n_pairs": 8,
70 "per_seed_diffs": [
71 -0.03644293546676636,
72 0.17473813891410828,
73 0.06784725189208984,
74 -0.18506783246994019,
75 0.08176279067993164,
76 0.002299785614013672,
77 0.11128050088882446,
78 -0.03985726833343506
79 ],
80 "p_value": 0.5933,
81 "mde": 0.09347444437257611,
82 "mde_rel_pct": 16.629457124352946,
83 "verdict": "no significant win",
84 "system_worked": false
85 },
86 "mechanism_signature": {
87 "mechanism_signature": {
88 "prediction": "identity insertion and O(eta) untied response",
89 "confirmed": true,
90 "idea_best_cfg": {
91 "lr": 0.0015,
92 "weight_decay": 0.0
93 },
94 "per_seed": [
95 {
96 "seed": 0,
97 "insertion_relative_output_change": 1.742963640083417e-08,
98 "trained_pair_residual": 0.013553540222346783,
99 "untied_eta_response": [
100 [
101 1e-05,
102 6.882298180244106e-07
103 ],
104 [
105 0.0001,
106 6.846436008345336e-06
107 ],
108 [
109 0.001,
110 6.854822277091444e-05
111 ]
112 ],
113 "trained_loglog_slope": 0.9991313573267615
114 },
115 {
116 "seed": 1,
117 "insertion_relative_output_change": 1.768672674984373e-08,
118 "trained_pair_residual": 0.01374770887196064,
119 "untied_eta_response": [
120 [
121 1e-05,
122 1.540886728434998e-06
123 ],
124 [
125 0.0001,
126 1.5476085536647588e-05
127 ],
128 [
129 0.001,
130 0.0001548135041957721
131 ]
132 ],
133 "trained_loglog_slope": 1.0010190631783853
134 },
135 {
136 "seed": 2,
137 "insertion_relative_output_change": 1.7788378769978408e-08,
138 "trained_pair_residual": 0.01247485727071762,
139 "untied_eta_response": [
140 [
141 1e-05,
142 2.696924639167264e-06
143 ],
144 [
145 0.0001,
146 2.6614063244778663e-05
147 ],
148 [
149 0.001,
150 0.0002659428573679179
151 ]
152 ],
153 "trained_loglog_slope": 0.9969597598038572
154 },
155 {
156 "seed": 3,
157 "insertion_relative_output_change": 1.678153260797899e-08,
158 "trained_pair_residual": 0.012198672629892826,
159 "untied_eta_response": [
160 [
161 1e-05,
162 3.872593197229435e-07
163 ],
164 [
165 0.0001,
166 3.2745517728471896e-06
167 ],
168 [
169 0.001,
170 3.2737443689256907e-05
171 ]
172 ],
173 "trained_loglog_slope": 0.9635214430977415
174 },
175 {
176 "seed": 4,
177 "insertion_relative_output_change": 1.788907688649033e-08,
178 "trained_pair_residual": 0.012925044633448124,
179 "untied_eta_response": [
180 [
181 1e-05,
182 1.7712729913910152e-06
183 ],
184 [
185 0.0001,
186 1.768083166098222e-05
187 ],
188 [
189 0.001,
190 0.00017684148042462766
191 ]
192 ],
193 "trained_loglog_slope": 0.9996493207179143
194 },
195 {
196 "seed": 5,
197 "insertion_relative_output_change": 1.6752988329926666e-08,
198 "trained_pair_residual": 0.011602357029914856,
199 "untied_eta_response": [
200 [
201 1e-05,
202 9.838264531936147e-07
203 ],
204 [
205 0.0001,
206 9.362133823742624e-06
207 ],
208 [
209 0.001,
210 9.372592467116192e-05
211 ]
212 ],
213 "trained_loglog_slope": 0.9894706190051281
214 },
215 {
216 "seed": 6,
217 "insertion_relative_output_change": 1.6823253901065982e-08,
218 "trained_pair_residual": 0.012010437436401844,
219 "untied_eta_response": [
220 [
221 1e-05,
222 1.4330839803733397e-06
223 ],
224 [
225 0.0001,
226 1.4343587281473447e-05
227 ],
228 [
229 0.001,
230 0.00014369039854500443
231 ]
232 ],
233 "trained_loglog_slope": 1.0005780540284896
234 },
235 {
236 "seed": 7,
237 "insertion_relative_output_change": 1.7179964117985946e-08,
238 "trained_pair_residual": 0.013055686838924885,
239 "untied_eta_response": [
240 [
241 1e-05,
242 6.872157314319338e-07
243 ],
244 [
245 0.0001,
246 6.427855623769574e-06
247 ],
248 [
249 0.001,
250 6.434921670006588e-05
251 ]
252 ],
253 "trained_loglog_slope": 0.9857250860758257
254 },
255 {
256 "seed": 0,
257 "insertion_relative_output_change": 1.742963640083417e-08,
258 "trained_pair_residual": 0.024646366015076637,
259 "untied_eta_response": [
260 [
261 1e-05,
262 6.151723823677457e-07
263 ],
264 [
265 0.0001,
266 6.120769739936804e-06
267 ],
268 [
269 0.001,
270 6.104174099164084e-05
271 ]
272 ],
273 "trained_loglog_slope": 0.9983150409119986
274 },
275 {
276 "seed": 1,
277 "insertion_relative_output_change": 1.768672674984373e-08,
278 "trained_pair_residual": 0.026445627212524414,
279 "untied_eta_response": [
280 [
281 1e-05,
282 1.0641194876370719e-06
283 ],
284 [
285 0.0001,
286 1.0895923878706526e-05
287 ],
288 [
289 0.001,
290 0.00010886218660743907
291 ]
292 ],
293 "trained_loglog_slope": 1.0049433283222116
294 },
295 {
296 "seed": 2,
297 "insertion_relative_output_change": 1.7788378769978408e-08,
298 "trained_pair_residual": 0.023685336112976074,
299 "untied_eta_response": [
300 [
301 1e-05,
302 1.0162499393118196e-06
303 ],
304 [
305 0.0001,
306 1.0251549610984512e-05
307 ],
308 [
309 0.001,
310 0.00010272490908391774
311 ]
312 ],
313 "trained_loglog_slope": 1.0023376164486255
314 },
315 {
316 "seed": 3,
317 "insertion_relative_output_change": 1.678153260797899e-08,
318 "trained_pair_residual": 0.024265628308057785,
319 "untied_eta_response": [
320 [
321 1e-05,
322 2.682033937162487e-06
323 ],
324 [
325 0.0001,
326 2.6792338758241385e-05
327 ],
328 [
329 0.001,
330 0.0002678318414837122
331 ]
332 ],
333 "trained_loglog_slope": 0.9996989692102766
334 },
335 {
336 "seed": 4,
337 "insertion_relative_output_change": 1.788907688649033e-08,
338 "trained_pair_residual": 0.02498849853873253,
339 "untied_eta_response": [
340 [
341 1e-05,
342 1.6889165408429108e-06
343 ],
344 [
345 0.0001,
346 1.6884097931324504e-05
347 ],
348 [
349 0.001,
350 0.00016868698003236204
351 ]
352 ],
353 "trained_loglog_slope": 0.9997366870611765
354 },
355 {
356 "seed": 5,
357 "insertion_relative_output_change": 1.6752988329926666e-08,
358 "trained_pair_residual": 0.02313663437962532,
359 "untied_eta_response": [
360 [
361 1e-05,
362 1.1504967005748767e-06
363 ],
364 [
365 0.0001,
366 1.1409663784434088e-05
367 ],
368 [
369 0.001,
370 0.00011402922245906666
371 ]
372 ],
373 "trained_loglog_slope": 0.9980653927231617
374 },
375 {
376 "seed": 6,
377 "insertion_relative_output_change": 1.6823253901065982e-08,
378 "trained_pair_residual": 0.023537755012512207,
379 "untied_eta_response": [
380 [
381 1e-05,
382 1.739130652822496e-06
383 ],
384 [
385 0.0001,
386 1.758110556693282e-05
387 ],
388 [
389 0.001,
390 0.00017618776473682374
391 ]
392 ],
393 "trained_loglog_slope": 1.0028217680103706
394 },
395 {
396 "seed": 7,
397 "insertion_relative_output_change": 1.7179964117985946e-08,
398 "trained_pair_residual": 0.024072429165244102,
399 "untied_eta_response": [
400 [
401 1e-05,
402 7.772475214551378e-07
403 ],
404 [
405 0.0001,
406 7.599023319926346e-06
407 ],
408 [
409 0.001,
410 7.574163464596495e-05
411 ]
412 ],
413 "trained_loglog_slope": 0.9943876639826758
414 },
415 {
416 "seed": 0,
417 "insertion_relative_output_change": 1.742963640083417e-08,
418 "trained_pair_residual": 0.049147915095090866,
419 "untied_eta_response": [
420 [
421 1e-05,
422 1.9303267890791176e-06
423 ],
424 [
425 0.0001,
426 1.8857945178751834e-05
427 ],
428 [
429 0.001,
430 0.0001881620701169595
431 ]
432 ],
433 "trained_loglog_slope": 0.9944506222877422
434 },
435 {
436 "seed": 1,
437 "insertion_relative_output_change": 1.768672674984373e-08,
438 "trained_pair_residual": 0.04954760521650314,
439 "untied_eta_response": [
440 [
441 1e-05,
442 8.311401870741975e-06
443 ],
444 [
445 0.0001,
446 8.277293818537146e-05
447 ],
448 [
449 0.001,
450 0.0008280269103124738
451 ]
452 ],
453 "trained_loglog_slope": 0.9991850847890872
454 },
455 {
456 "seed": 2,
457 "insertion_relative_output_change": 1.7788378769978408e-08,
458 "trained_pair_residual": 0.04762578755617142,
459 "untied_eta_response": [
460 [
461 1e-05,
462 7.512046067859046e-06
463 ],
464 [
465 0.0001,
466 7.488950359402224e-05
467 ],
468 [
469 0.001,
470 0.0007490166462957859
471 ]
472 ],
473 "trained_loglog_slope": 0.9993666135316805
474 },
475 {
476 "seed": 3,
477 "insertion_relative_output_change": 1.678153260797899e-08,
478 "trained_pair_residual": 0.045538078993558884,
479 "untied_eta_response": [
480 [
481 1e-05,
482 1.7467951693106443e-05
483 ],
484 [
485 0.0001,
486 0.000174284607055597
487 ],
488 [
489 0.001,
490 0.0017429785802960396
491 ]
492 ],
493 "trained_loglog_slope": 0.9995250339052046
494 },
495 {
496 "seed": 4,
497 "insertion_relative_output_change": 1.788907688649033e-08,
498 "trained_pair_residual": 0.056724853813648224,
499 "untied_eta_response": [
500 [
501 1e-05,
502 4.201423053018516e-06
503 ],
504 [
505 0.0001,
506 4.1662235162220895e-05
507 ],
508 [
509 0.001,
510 0.00041710384539328516
511 ]
512 ],
513 "trained_loglog_slope": 0.9984238898394366
514 },
515 {
516 "seed": 5,
517 "insertion_relative_output_change": 1.6752988329926666e-08,
518 "trained_pair_residual": 0.046788010746240616,
519 "untied_eta_response": [
520 [
521 1e-05,
522 5.517295448953519e-06
523 ],
524 [
525 0.0001,
526 5.510038317879662e-05
527 ],
528 [
529 0.001,
530 0.0005517865065485239
531 ]
532 ],
533 "trained_loglog_slope": 1.000022417554908
534 },
535 {
536 "seed": 6,
537 "insertion_relative_output_change": 1.6823253901065982e-08,
538 "trained_pair_residual": 0.04568944126367569,
539 "untied_eta_response": [
540 [
541 1e-05,
542 5.431155841506552e-06
543 ],
544 [
545 0.0001,
546 5.4161326261237264e-05
547 ],
548 [
549 0.001,
550 0.0005412694299593568
551 ]
552 ],
553 "trained_loglog_slope": 0.9992606174539198
554 },
555 {
556 "seed": 7,
557 "insertion_relative_output_change": 1.7179964117985946e-08,
558 "trained_pair_residual": 0.04901719093322754,
559 "untied_eta_response": [
560 [
561 1e-05,
562 5.4369570534618106e-06
563 ],
564 [
565 0.0001,
566 5.419322042143904e-05
567 ],
568 [
569 0.001,
570 0.0005417013308033347
571 ]
572 ],
573 "trained_loglog_slope": 0.9992020000011609
574 }
575 ],
576 "mean_insertion_relative_output_change": 1.7291444720513027e-08,
577 "mean_trained_pair_residual": 0.028601060970686376
578 }
579 }
580}