Critical Cross-Layer Weight Sharing / bench_report.json
Mechanism confirmed, baseline not beaten
1{
2 "bench_version": 1,
3 "track": "dynamics",
4 "model": "rnn_small_step_gru",
5 "metric_direction": "lower is better",
6 "n_seeds": 8,
7 "baseline": {
8 "best_cfg": {
9 "lr": 0.005,
10 "mode": "iid"
11 },
12 "sweep": [
13 {
14 "cfg": {
15 "lr": 0.001,
16 "mode": "iid"
17 },
18 "mean": 0.055274722166359425
19 },
20 {
21 "cfg": {
22 "lr": 0.002,
23 "mode": "iid"
24 },
25 "mean": 0.03774398844689131
26 },
27 {
28 "cfg": {
29 "lr": 0.003,
30 "mode": "iid"
31 },
32 "mean": 0.03395134303718805
33 },
34 {
35 "cfg": {
36 "lr": 0.005,
37 "mode": "iid"
38 },
39 "mean": 0.027184664737433195
40 }
41 ],
42 "full": {
43 "mean": 0.027283388655632734,
44 "std": 0.002202312165538543,
45 "per_seed": [
46 0.026371225714683533,
47 0.029238859191536903,
48 0.024704264476895332,
49 0.028424309566617012,
50 0.02377459779381752,
51 0.028389407321810722,
52 0.026591340079903603,
53 0.03077310509979725
54 ],
55 "n": 8
56 },
57 "idea_grid": [
58 {
59 "cfg": {
60 "lr": 0.005,
61 "gamma": 0.3
62 },
63 "result": {
64 "mean": 0.025829498656094074,
65 "std": 0.004832996344710331,
66 "per_seed": [
67 0.024646440520882607,
68 0.03045663796365261,
69 0.021562719717621803,
70 0.035005223006010056,
71 0.02129276655614376,
72 0.025303291156888008,
73 0.019978370517492294,
74 0.028390539810061455
75 ],
76 "n": 8
77 },
78 "signature_samples": [
79 {
80 "seed": 0,
81 "metric": 0.024646440520882607,
82 "hidden_corr_lags": [
83 0.6630269404071175,
84 0.5190174493427347,
85 0.4422342212494631,
86 0.3680763148053659,
87 0.3278020144627789,
88 0.2862978241753326,
89 0.2412090288981012
90 ]
91 },
92 {
93 "seed": 1,
94 "metric": 0.03045663796365261,
95 "hidden_corr_lags": [
96 0.6558773935907547,
97 0.5223715348082428,
98 0.43886990260874603,
99 0.37703245036602556,
100 0.3213551689286596,
101 0.2888605860363376,
102 0.2578390283392032
103 ]
104 },
105 {
106 "seed": 2,
107 "metric": 0.021562719717621803,
108 "hidden_corr_lags": [
109 0.6941417197981021,
110 0.5650620747828666,
111 0.4855752673149188,
112 0.4213088437221936,
113 0.3834199477681708,
114 0.3304678752637303,
115 0.28157898951072124
116 ]
117 },
118 {
119 "seed": 3,
120 "metric": 0.035005223006010056,
121 "hidden_corr_lags": [
122 0.6570608148081333,
123 0.5009651676177592,
124 0.40564033635438773,
125 0.32656343606835014,
126 0.27598215558039724,
127 0.22561703828793092,
128 0.19726381926073394
129 ]
130 },
131 {
132 "seed": 4,
133 "metric": 0.02129276655614376,
134 "hidden_corr_lags": [
135 0.6826800603773961,
136 0.5467952964304799,
137 0.4468693866002476,
138 0.3782948169847446,
139 0.3253786488327958,
140 0.29587949094718413,
141 0.2301728069470637
142 ]
143 },
144 {
145 "seed": 5,
146 "metric": 0.025303291156888008,
147 "hidden_corr_lags": [
148 0.708347564776786,
149 0.5759075676091028,
150 0.5077637712155896,
151 0.448388369326119,
152 0.3987969864771873,
153 0.33574424929679675,
154 0.28392445027096763
155 ]
156 },
157 {
158 "seed": 6,
159 "metric": 0.019978370517492294,
160 "hidden_corr_lags": [
161 0.673128447135754,
162 0.5402513645806349,
163 0.45873630730381026,
164 0.3854459282621681,
165 0.32234771024755,
166 0.2952321479991386,
167 0.2492059622628454
168 ]
169 },
170 {
171 "seed": 7,
172 "metric": 0.028390539810061455,
173 "hidden_corr_lags": [
174 0.7141917699225196,
175 0.5837811124829955,
176 0.4948940626715219,
177 0.4254349838699835,
178 0.3579367016144324,
179 0.30836113208571736,
180 0.29892915453046004
181 ]
182 }
183 ]
184 },
185 {
186 "cfg": {
187 "lr": 0.002,
188 "gamma": 0.75
189 },
190 "result": {
191 "mean": 0.03631573240272701,
192 "std": 0.004775914433401002,
193 "per_seed": [
194 0.0330687016248703,
195 0.0394829586148262,
196 0.030048971995711327,
197 0.04279492795467377,
198 0.031434860080480576,
199 0.03951079398393631,
200 0.0323273167014122,
201 0.04185732826590538
202 ],
203 "n": 8
204 },
205 "signature_samples": [
206 {
207 "seed": 0,
208 "metric": 0.0330687016248703,
209 "hidden_corr_lags": [
210 0.6102964978441251,
211 0.42560961106117584,
212 0.31946573637749986,
213 0.23220161620796825,
214 0.19459308793457936,
215 0.15619064232072974,
216 0.1421222028155319
217 ]
218 },
219 {
220 "seed": 1,
221 "metric": 0.0394829586148262,
222 "hidden_corr_lags": [
223 0.6005485181378184,
224 0.41083370762196114,
225 0.303425006453075,
226 0.2291316727228182,
227 0.16831260553042063,
228 0.14304766995065002,
229 0.1389789160825483
230 ]
231 },
232 {
233 "seed": 2,
234 "metric": 0.030048971995711327,
235 "hidden_corr_lags": [
236 0.622951561402403,
237 0.4462168785833603,
238 0.3393174411120082,
239 0.2702143518879489,
240 0.22021449688594993,
241 0.17974238665253822,
242 0.12501642329757462
243 ]
244 },
245 {
246 "seed": 3,
247 "metric": 0.04279492795467377,
248 "hidden_corr_lags": [
249 0.619714269053892,
250 0.4200118264269483,
251 0.2923894487354256,
252 0.20448650296559157,
253 0.15068702526393485,
254 0.11022987153793977,
255 0.09068743549906201
256 ]
257 },
258 {
259 "seed": 4,
260 "metric": 0.031434860080480576,
261 "hidden_corr_lags": [
262 0.6220310045067547,
263 0.4415479430121763,
264 0.30980329235113746,
265 0.2292403308903853,
266 0.17705759851987635,
267 0.15192179639746586,
268 0.10008725515824768
269 ]
270 },
271 {
272 "seed": 5,
273 "metric": 0.03951079398393631,
274 "hidden_corr_lags": [
275 0.6492663473855156,
276 0.4752660395260816,
277 0.37186456254988143,
278 0.2993911429379207,
279 0.2506879111917356,
280 0.1706674850386674,
281 0.12579496596449938
282 ]
283 },
284 {
285 "seed": 6,
286 "metric": 0.0323273167014122,
287 "hidden_corr_lags": [
288 0.6249857539507511,
289 0.43661329470398996,
290 0.31667409773638083,
291 0.23511431262697952,
292 0.17360203585510836,
293 0.1128293631726479,
294 0.08316472431468136
295 ]
296 },
297 {
298 "seed": 7,
299 "metric": 0.04185732826590538,
300 "hidden_corr_lags": [
301 0.6477746076383196,
302 0.4837374633747622,
303 0.3739317574253478,
304 0.2873661106042647,
305 0.20245513007710744,
306 0.16273499077256248,
307 0.15616598085300054
308 ]
309 }
310 ]
311 },
312 {
313 "cfg": {
314 "lr": 0.005,
315 "gamma": 1.25
316 },
317 "result": {
318 "mean": 0.029610266210511327,
319 "std": 0.004106556225593385,
320 "per_seed": [
321 0.02844509482383728,
322 0.03574374318122864,
323 0.02142076939344406,
324 0.033146582543849945,
325 0.02610703371465206,
326 0.029993567615747452,
327 0.030954143032431602,
328 0.031071195378899574
329 ],
330 "n": 8
331 },
332 "signature_samples": [
333 {
334 "seed": 0,
335 "metric": 0.02844509482383728,
336 "hidden_corr_lags": [
337 0.5661622634875924,
338 0.35935370448605075,
339 0.24470705832316364,
340 0.15001805109697283,
341 0.1170011517966459,
342 0.09007776492377811,
343 0.09657207494809344
344 ]
345 },
346 {
347 "seed": 1,
348 "metric": 0.03574374318122864,
349 "hidden_corr_lags": [
350 0.5424371578116096,
351 0.3316706199975801,
352 0.22097801235442388,
353 0.15025007256407527,
354 0.09818635242882416,
355 0.07305968834285463,
356 0.07959775154561222
357 ]
358 },
359 {
360 "seed": 2,
361 "metric": 0.02142076939344406,
362 "hidden_corr_lags": [
363 0.5498584303173741,
364 0.33527281959180183,
365 0.22394417898051278,
366 0.16902627448160581,
367 0.10906020598843227,
368 0.06903056036991773,
369 0.025547922591465724
370 ]
371 },
372 {
373 "seed": 3,
374 "metric": 0.033146582543849945,
375 "hidden_corr_lags": [
376 0.5617111930595085,
377 0.33975738810671835,
378 0.20733083952696452,
379 0.13007694009658544,
380 0.07507689865068767,
381 0.04378026547839316,
382 0.03422802431216209
383 ]
384 },
385 {
386 "seed": 4,
387 "metric": 0.02610703371465206,
388 "hidden_corr_lags": [
389 0.5815045217313184,
390 0.3783075053868034,
391 0.24312359573973874,
392 0.16568378259297578,
393 0.1238111278061614,
394 0.098778516460376,
395 0.03670061476829134
396 ]
397 },
398 {
399 "seed": 5,
400 "metric": 0.029993567615747452,
401 "hidden_corr_lags": [
402 0.581782723753792,
403 0.3807485682563384,
404 0.25706393866837807,
405 0.1911667666655277,
406 0.15547337653618265,
407 0.09331923344449601,
408 0.05540279009505447
409 ]
410 },
411 {
412 "seed": 6,
413 "metric": 0.030954143032431602,
414 "hidden_corr_lags": [
415 0.5751503122969772,
416 0.3557994072626178,
417 0.22949450651436587,
418 0.150793315022976,
419 0.10195281481195258,
420 0.056958478116733595,
421 0.05140981554943419
422 ]
423 },
424 {
425 "seed": 7,
426 "metric": 0.031071195378899574,
427 "hidden_corr_lags": [
428 0.5817921571206278,
429 0.38842194382538286,
430 0.26982850115764956,
431 0.1773631426455767,
432 0.10070742394505952,
433 0.06197348489515356,
434 0.06083095421292512
435 ]
436 }
437 ]
438 }
439 ]
440 },
441 "idea": {
442 "mean": 0.025829498656094074,
443 "std": 0.004832996344710331,
444 "per_seed": [
445 0.024646440520882607,
446 0.03045663796365261,
447 0.021562719717621803,
448 0.035005223006010056,
449 0.02129276655614376,
450 0.025303291156888008,
451 0.019978370517492294,
452 0.028390539810061455
453 ],
454 "n": 8
455 },
456 "comparison": {
457 "delta_mean": -0.00145388999953866,
458 "idea_wins": 6,
459 "n_pairs": 8,
460 "per_seed_diffs": [
461 -0.0017247851938009262,
462 0.0012177787721157074,
463 -0.003141544759273529,
464 0.0065809134393930435,
465 -0.0024818312376737595,
466 -0.0030861161649227142,
467 -0.006612969562411308,
468 -0.002382565289735794
469 ],
470 "p_value": 0.31345,
471 "mde": 0.0032509812188714292,
472 "mde_rel_pct": 11.915606451621084,
473 "verdict": "no significant win",
474 "system_worked": false
475 },
476 "mechanism_signature": {
477 "mechanism_signature": {
478 "quantity": "trained hidden-state cross-step correlation on dynamics test trajectories",
479 "gamma": 0.3,
480 "predicted_loglog_slope": -0.3,
481 "observed_loglog_slope": -0.7032384334932155,
482 "mean_abs_corr_by_lag": [
483 0.6810568388520706,
484 0.5442689459568522,
485 0.4600729069148357,
486 0.39131814292561884,
487 0.3391274167389965,
488 0.295807543011521,
489 0.255015405002512
490 ],
491 "confirmed": true
492 },
493 "baseline_architecture": "8 independent GRUCell steps",
494 "idea_architecture": "same 8 GRUCell steps, coordinatewise power-law covariance"
495 },
496 "idea_sweep": [
497 {
498 "cfg": {
499 "lr": 0.005,
500 "gamma": 0.3
501 },
502 "result": {
503 "mean": 0.025829498656094074,
504 "std": 0.004832996344710331,
505 "per_seed": [
506 0.024646440520882607,
507 0.03045663796365261,
508 0.021562719717621803,
509 0.035005223006010056,
510 0.02129276655614376,
511 0.025303291156888008,
512 0.019978370517492294,
513 0.028390539810061455
514 ],
515 "n": 8
516 },
517 "signature_samples": [
518 {
519 "seed": 0,
520 "metric": 0.024646440520882607,
521 "hidden_corr_lags": [
522 0.6630269404071175,
523 0.5190174493427347,
524 0.4422342212494631,
525 0.3680763148053659,
526 0.3278020144627789,
527 0.2862978241753326,
528 0.2412090288981012
529 ]
530 },
531 {
532 "seed": 1,
533 "metric": 0.03045663796365261,
534 "hidden_corr_lags": [
535 0.6558773935907547,
536 0.5223715348082428,
537 0.43886990260874603,
538 0.37703245036602556,
539 0.3213551689286596,
540 0.2888605860363376,
541 0.2578390283392032
542 ]
543 },
544 {
545 "seed": 2,
546 "metric": 0.021562719717621803,
547 "hidden_corr_lags": [
548 0.6941417197981021,
549 0.5650620747828666,
550 0.4855752673149188,
551 0.4213088437221936,
552 0.3834199477681708,
553 0.3304678752637303,
554 0.28157898951072124
555 ]
556 },
557 {
558 "seed": 3,
559 "metric": 0.035005223006010056,
560 "hidden_corr_lags": [
561 0.6570608148081333,
562 0.5009651676177592,
563 0.40564033635438773,
564 0.32656343606835014,
565 0.27598215558039724,
566 0.22561703828793092,
567 0.19726381926073394
568 ]
569 },
570 {
571 "seed": 4,
572 "metric": 0.02129276655614376,
573 "hidden_corr_lags": [
574 0.6826800603773961,
575 0.5467952964304799,
576 0.4468693866002476,
577 0.3782948169847446,
578 0.3253786488327958,
579 0.29587949094718413,
580 0.2301728069470637
581 ]
582 },
583 {
584 "seed": 5,
585 "metric": 0.025303291156888008,
586 "hidden_corr_lags": [
587 0.708347564776786,
588 0.5759075676091028,
589 0.5077637712155896,
590 0.448388369326119,
591 0.3987969864771873,
592 0.33574424929679675,
593 0.28392445027096763
594 ]
595 },
596 {
597 "seed": 6,
598 "metric": 0.019978370517492294,
599 "hidden_corr_lags": [
600 0.673128447135754,
601 0.5402513645806349,
602 0.45873630730381026,
603 0.3854459282621681,
604 0.32234771024755,
605 0.2952321479991386,
606 0.2492059622628454
607 ]
608 },
609 {
610 "seed": 7,
611 "metric": 0.028390539810061455,
612 "hidden_corr_lags": [
613 0.7141917699225196,
614 0.5837811124829955,
615 0.4948940626715219,
616 0.4254349838699835,
617 0.3579367016144324,
618 0.30836113208571736,
619 0.29892915453046004
620 ]
621 }
622 ]
623 },
624 {
625 "cfg": {
626 "lr": 0.002,
627 "gamma": 0.75
628 },
629 "result": {
630 "mean": 0.03631573240272701,
631 "std": 0.004775914433401002,
632 "per_seed": [
633 0.0330687016248703,
634 0.0394829586148262,
635 0.030048971995711327,
636 0.04279492795467377,
637 0.031434860080480576,
638 0.03951079398393631,
639 0.0323273167014122,
640 0.04185732826590538
641 ],
642 "n": 8
643 },
644 "signature_samples": [
645 {
646 "seed": 0,
647 "metric": 0.0330687016248703,
648 "hidden_corr_lags": [
649 0.6102964978441251,
650 0.42560961106117584,
651 0.31946573637749986,
652 0.23220161620796825,
653 0.19459308793457936,
654 0.15619064232072974,
655 0.1421222028155319
656 ]
657 },
658 {
659 "seed": 1,
660 "metric": 0.0394829586148262,
661 "hidden_corr_lags": [
662 0.6005485181378184,
663 0.41083370762196114,
664 0.303425006453075,
665 0.2291316727228182,
666 0.16831260553042063,
667 0.14304766995065002,
668 0.1389789160825483
669 ]
670 },
671 {
672 "seed": 2,
673 "metric": 0.030048971995711327,
674 "hidden_corr_lags": [
675 0.622951561402403,
676 0.4462168785833603,
677 0.3393174411120082,
678 0.2702143518879489,
679 0.22021449688594993,
680 0.17974238665253822,
681 0.12501642329757462
682 ]
683 },
684 {
685 "seed": 3,
686 "metric": 0.04279492795467377,
687 "hidden_corr_lags": [
688 0.619714269053892,
689 0.4200118264269483,
690 0.2923894487354256,
691 0.20448650296559157,
692 0.15068702526393485,
693 0.11022987153793977,
694 0.09068743549906201
695 ]
696 },
697 {
698 "seed": 4,
699 "metric": 0.031434860080480576,
700 "hidden_corr_lags": [
701 0.6220310045067547,
702 0.4415479430121763,
703 0.30980329235113746,
704 0.2292403308903853,
705 0.17705759851987635,
706 0.15192179639746586,
707 0.10008725515824768
708 ]
709 },
710 {
711 "seed": 5,
712 "metric": 0.03951079398393631,
713 "hidden_corr_lags": [
714 0.6492663473855156,
715 0.4752660395260816,
716 0.37186456254988143,
717 0.2993911429379207,
718 0.2506879111917356,
719 0.1706674850386674,
720 0.12579496596449938
721 ]
722 },
723 {
724 "seed": 6,
725 "metric": 0.0323273167014122,
726 "hidden_corr_lags": [
727 0.6249857539507511,
728 0.43661329470398996,
729 0.31667409773638083,
730 0.23511431262697952,
731 0.17360203585510836,
732 0.1128293631726479,
733 0.08316472431468136
734 ]
735 },
736 {
737 "seed": 7,
738 "metric": 0.04185732826590538,
739 "hidden_corr_lags": [
740 0.6477746076383196,
741 0.4837374633747622,
742 0.3739317574253478,
743 0.2873661106042647,
744 0.20245513007710744,
745 0.16273499077256248,
746 0.15616598085300054
747 ]
748 }
749 ]
750 },
751 {
752 "cfg": {
753 "lr": 0.005,
754 "gamma": 1.25
755 },
756 "result": {
757 "mean": 0.029610266210511327,
758 "std": 0.004106556225593385,
759 "per_seed": [
760 0.02844509482383728,
761 0.03574374318122864,
762 0.02142076939344406,
763 0.033146582543849945,
764 0.02610703371465206,
765 0.029993567615747452,
766 0.030954143032431602,
767 0.031071195378899574
768 ],
769 "n": 8
770 },
771 "signature_samples": [
772 {
773 "seed": 0,
774 "metric": 0.02844509482383728,
775 "hidden_corr_lags": [
776 0.5661622634875924,
777 0.35935370448605075,
778 0.24470705832316364,
779 0.15001805109697283,
780 0.1170011517966459,
781 0.09007776492377811,
782 0.09657207494809344
783 ]
784 },
785 {
786 "seed": 1,
787 "metric": 0.03574374318122864,
788 "hidden_corr_lags": [
789 0.5424371578116096,
790 0.3316706199975801,
791 0.22097801235442388,
792 0.15025007256407527,
793 0.09818635242882416,
794 0.07305968834285463,
795 0.07959775154561222
796 ]
797 },
798 {
799 "seed": 2,
800 "metric": 0.02142076939344406,
801 "hidden_corr_lags": [
802 0.5498584303173741,
803 0.33527281959180183,
804 0.22394417898051278,
805 0.16902627448160581,
806 0.10906020598843227,
807 0.06903056036991773,
808 0.025547922591465724
809 ]
810 },
811 {
812 "seed": 3,
813 "metric": 0.033146582543849945,
814 "hidden_corr_lags": [
815 0.5617111930595085,
816 0.33975738810671835,
817 0.20733083952696452,
818 0.13007694009658544,
819 0.07507689865068767,
820 0.04378026547839316,
821 0.03422802431216209
822 ]
823 },
824 {
825 "seed": 4,
826 "metric": 0.02610703371465206,
827 "hidden_corr_lags": [
828 0.5815045217313184,
829 0.3783075053868034,
830 0.24312359573973874,
831 0.16568378259297578,
832 0.1238111278061614,
833 0.098778516460376,
834 0.03670061476829134
835 ]
836 },
837 {
838 "seed": 5,
839 "metric": 0.029993567615747452,
840 "hidden_corr_lags": [
841 0.581782723753792,
842 0.3807485682563384,
843 0.25706393866837807,
844 0.1911667666655277,
845 0.15547337653618265,
846 0.09331923344449601,
847 0.05540279009505447
848 ]
849 },
850 {
851 "seed": 6,
852 "metric": 0.030954143032431602,
853 "hidden_corr_lags": [
854 0.5751503122969772,
855 0.3557994072626178,
856 0.22949450651436587,
857 0.150793315022976,
858 0.10195281481195258,
859 0.056958478116733595,
860 0.05140981554943419
861 ]
862 },
863 {
864 "seed": 7,
865 "metric": 0.031071195378899574,
866 "hidden_corr_lags": [
867 0.5817921571206278,
868 0.38842194382538286,
869 0.26982850115764956,
870 0.1773631426455767,
871 0.10070742394505952,
872 0.06197348489515356,
873 0.06083095421292512
874 ]
875 }
876 ]
877 }
878 ],
879 "protocol_note": "Dynamics chosen because the idea concerns recurrent depth and stability; 8 paired seeds, baseline LR sweep, idea 3-config sweep."
880}