Recursive Nonlocal Edge Feedback GNN / bench_report.json
Beats tuned baseline
1{
2 "bench_version": 1,
3 "track": "dynamics",
4 "model": "rnn_small",
5 "metric_direction": "lower is better",
6 "n_seeds": 8,
7 "baseline": {
8 "sweep": [
9 {
10 "config": {
11 "lr": 0.0015,
12 "weight_decay": 0.0
13 },
14 "per_seed": [
15 0.0008893852354958653,
16 0.0011461004614830017,
17 0.0009290041634812951,
18 0.0013412678381428123
19 ],
20 "mean": 0.0010764394246507436,
21 "std": 0.00018146154865546845,
22 "n": 4,
23 "details": [
24 {
25 "seed": 0,
26 "metric": 0.0008893852354958653,
27 "params": 13313,
28 "last_loss": 0.0009362917765974999
29 },
30 {
31 "seed": 1,
32 "metric": 0.0011461004614830017,
33 "params": 13313,
34 "last_loss": 0.0009171083651017398
35 },
36 {
37 "seed": 2,
38 "metric": 0.0009290041634812951,
39 "params": 13313,
40 "last_loss": 0.0008064234256744385
41 },
42 {
43 "seed": 3,
44 "metric": 0.0013412678381428123,
45 "params": 13313,
46 "last_loss": 0.000837117291521281
47 }
48 ]
49 },
50 {
51 "config": {
52 "lr": 0.003,
53 "weight_decay": 0.0
54 },
55 "per_seed": [
56 0.0006203416851349175,
57 0.0007658391259610653,
58 0.0006099292659200728,
59 0.0009500968153588474
60 ],
61 "mean": 0.0007365517230937257,
62 "std": 0.0001378380190480065,
63 "n": 4,
64 "details": [
65 {
66 "seed": 0,
67 "metric": 0.0006203416851349175,
68 "params": 13313,
69 "last_loss": 0.000665924958884716
70 },
71 {
72 "seed": 1,
73 "metric": 0.0007658391259610653,
74 "params": 13313,
75 "last_loss": 0.0007195674593094737
76 },
77 {
78 "seed": 2,
79 "metric": 0.0006099292659200728,
80 "params": 13313,
81 "last_loss": 0.0005001857841853052
82 },
83 {
84 "seed": 3,
85 "metric": 0.0009500968153588474,
86 "params": 13313,
87 "last_loss": 0.00065102310734801
88 }
89 ]
90 },
91 {
92 "config": {
93 "lr": 0.006,
94 "weight_decay": 0.0
95 },
96 "per_seed": [
97 0.00043351209023967385,
98 0.0006403940496966243,
99 0.0003332794294692576,
100 0.0005523009458556771
101 ],
102 "mean": 0.0004898716288153082,
103 "std": 0.00011646022207023158,
104 "n": 4,
105 "details": [
106 {
107 "seed": 0,
108 "metric": 0.00043351209023967385,
109 "params": 13313,
110 "last_loss": 0.0004344877786934376
111 },
112 {
113 "seed": 1,
114 "metric": 0.0006403940496966243,
115 "params": 13313,
116 "last_loss": 0.000438494966365397
117 },
118 {
119 "seed": 2,
120 "metric": 0.0003332794294692576,
121 "params": 13313,
122 "last_loss": 0.00030466696247458456
123 },
124 {
125 "seed": 3,
126 "metric": 0.0005523009458556771,
127 "params": 13313,
128 "last_loss": 0.0003513497015228495
129 }
130 ]
131 },
132 {
133 "config": {
134 "lr": 0.003,
135 "weight_decay": 0.0001
136 },
137 "per_seed": [
138 0.0006000761059112847,
139 0.0007683035801164806,
140 0.0006060203886590898,
141 0.0009272810420952737
142 ],
143 "mean": 0.0007254202791955322,
144 "std": 0.00013467947201853758,
145 "n": 4,
146 "details": [
147 {
148 "seed": 0,
149 "metric": 0.0006000761059112847,
150 "params": 13313,
151 "last_loss": 0.0006504375394433737
152 },
153 {
154 "seed": 1,
155 "metric": 0.0007683035801164806,
156 "params": 13313,
157 "last_loss": 0.0007221100921742618
158 },
159 {
160 "seed": 2,
161 "metric": 0.0006060203886590898,
162 "params": 13313,
163 "last_loss": 0.0004942747787572443
164 },
165 {
166 "seed": 3,
167 "metric": 0.0009272810420952737,
168 "params": 13313,
169 "last_loss": 0.000634368343744427
170 }
171 ]
172 }
173 ],
174 "best_config": {
175 "lr": 0.006,
176 "weight_decay": 0.0
177 },
178 "full": {
179 "config": {
180 "lr": 0.006,
181 "weight_decay": 0.0
182 },
183 "per_seed": [
184 0.00043351209023967385,
185 0.0006403940496966243,
186 0.0003332794294692576,
187 0.0005523009458556771,
188 0.00037599660572595894,
189 0.0005211277748458087,
190 0.00046043944894336164,
191 0.0004712840309366584
192 ],
193 "mean": 0.00047354179696412757,
194 "std": 9.170253921476272e-05,
195 "n": 8,
196 "details": [
197 {
198 "seed": 0,
199 "metric": 0.00043351209023967385,
200 "params": 13313,
201 "last_loss": 0.0004344877786934376
202 },
203 {
204 "seed": 1,
205 "metric": 0.0006403940496966243,
206 "params": 13313,
207 "last_loss": 0.000438494966365397
208 },
209 {
210 "seed": 2,
211 "metric": 0.0003332794294692576,
212 "params": 13313,
213 "last_loss": 0.00030466696247458456
214 },
215 {
216 "seed": 3,
217 "metric": 0.0005523009458556771,
218 "params": 13313,
219 "last_loss": 0.0003513497015228495
220 },
221 {
222 "seed": 4,
223 "metric": 0.00037599660572595894,
224 "params": 13313,
225 "last_loss": 0.0005308800761122256
226 },
227 {
228 "seed": 5,
229 "metric": 0.0005211277748458087,
230 "params": 13313,
231 "last_loss": 0.0005809032125398516
232 },
233 {
234 "seed": 6,
235 "metric": 0.00046043944894336164,
236 "params": 13313,
237 "last_loss": 0.00040943760424852374
238 },
239 {
240 "seed": 7,
241 "metric": 0.0004712840309366584,
242 "params": 13313,
243 "last_loss": 0.00035525997867807746
244 }
245 ]
246 }
247 },
248 "idea": {
249 "config": {
250 "lr": 0.006,
251 "weight_decay": 0.0
252 },
253 "per_seed": [
254 0.00012686560512520373,
255 0.0004387780209071934,
256 0.0002368502609897405,
257 0.00015413027722388506,
258 0.00017462033429183066,
259 0.00011361205542925745,
260 0.0002228845260106027,
261 0.0003435298858676106
262 ],
263 "mean": 0.0002264088707306655,
264 "std": 0.00010575777904307542,
265 "n": 8,
266 "details": [
267 {
268 "seed": 0,
269 "metric": 0.00012686560512520373,
270 "params": 45857,
271 "last_loss": 0.00018608432030305265
272 },
273 {
274 "seed": 1,
275 "metric": 0.0004387780209071934,
276 "params": 45857,
277 "last_loss": 0.00024508837144821883
278 },
279 {
280 "seed": 2,
281 "metric": 0.0002368502609897405,
282 "params": 45857,
283 "last_loss": 0.0001869376830291003
284 },
285 {
286 "seed": 3,
287 "metric": 0.00015413027722388506,
288 "params": 45857,
289 "last_loss": 0.00015381897304905578
290 },
291 {
292 "seed": 4,
293 "metric": 0.00017462033429183066,
294 "params": 45857,
295 "last_loss": 0.00021269463730277493
296 },
297 {
298 "seed": 5,
299 "metric": 0.00011361205542925745,
300 "params": 45857,
301 "last_loss": 0.0001970260846428573
302 },
303 {
304 "seed": 6,
305 "metric": 0.0002228845260106027,
306 "params": 45857,
307 "last_loss": 0.00033824800048023464
308 },
309 {
310 "seed": 7,
311 "metric": 0.0003435298858676106,
312 "params": 45857,
313 "last_loss": 0.0003234831592999399
314 }
315 ]
316 },
317 "comparison": {
318 "delta_mean": -0.00024713292623346206,
319 "idea_wins": 8,
320 "n_pairs": 8,
321 "per_seed_diffs": [
322 -0.0003066464851144701,
323 -0.00020161602878943086,
324 -9.64291684795171e-05,
325 -0.00039817066863179207,
326 -0.00020137627143412828,
327 -0.0004075157194165513,
328 -0.00023755492293275893,
329 -0.0001277541450690478
330 ],
331 "p_value": 0.0081,
332 "mde": 9.65319348661709e-05,
333 "mde_rel_pct": 20.385092822858788,
334 "verdict": "idea better (significant)",
335 "system_worked": true
336 },
337 "mechanism_signature": {
338 "task": "trained dynamics test input sensitivity",
339 "predicted": "recursive feedback should give measurable sensitivity to non-adjacent tokens and bounded local sensitivity",
340 "observed": {
341 "nonlocal_input_grad_norm": 0.26336461305618286,
342 "final_token_grad_norm": 0.570110559463501,
343 "input_jacobian_norm": 0.628002405166626
344 },
345 "confirmed": true
346 },
347 "idea_sweep": [
348 {
349 "config": {
350 "lr": 0.0015,
351 "weight_decay": 0.0
352 },
353 "per_seed": [
354 0.0006614502635784447,
355 0.0005860325763933361,
356 0.0005585338221862912,
357 0.001089978963136673,
358 0.0005720907356590033,
359 0.00043726706644520164,
360 0.000829867203719914,
361 0.0007765801274217665
362 ],
363 "mean": 0.0006889750948175788,
364 "std": 0.00019161090766784528,
365 "n": 8,
366 "details": [
367 {
368 "seed": 0,
369 "metric": 0.0006614502635784447,
370 "params": 45857,
371 "last_loss": 0.0008753743558190763
372 },
373 {
374 "seed": 1,
375 "metric": 0.0005860325763933361,
376 "params": 45857,
377 "last_loss": 0.0004818156384862959
378 },
379 {
380 "seed": 2,
381 "metric": 0.0005585338221862912,
382 "params": 45857,
383 "last_loss": 0.0004933090682607144
384 },
385 {
386 "seed": 3,
387 "metric": 0.001089978963136673,
388 "params": 45857,
389 "last_loss": 0.0005436877929605544
390 },
391 {
392 "seed": 4,
393 "metric": 0.0005720907356590033,
394 "params": 45857,
395 "last_loss": 0.0005770999426022172
396 },
397 {
398 "seed": 5,
399 "metric": 0.00043726706644520164,
400 "params": 45857,
401 "last_loss": 0.00048569669714197515
402 },
403 {
404 "seed": 6,
405 "metric": 0.000829867203719914,
406 "params": 45857,
407 "last_loss": 0.0008137347456067801
408 },
409 {
410 "seed": 7,
411 "metric": 0.0007765801274217665,
412 "params": 45857,
413 "last_loss": 0.0005952932010404766
414 }
415 ]
416 },
417 {
418 "config": {
419 "lr": 0.003,
420 "weight_decay": 0.0
421 },
422 "per_seed": [
423 0.0006935568526387215,
424 0.0004945265245623887,
425 0.00023093193885870278,
426 0.0005860979435965419,
427 0.00023517308000009507,
428 0.00020430154108908027,
429 0.000541141489520669,
430 0.0005798843340016901
431 ],
432 "mean": 0.00044570171303348616,
433 "std": 0.0001800566802955679,
434 "n": 8,
435 "details": [
436 {
437 "seed": 0,
438 "metric": 0.0006935568526387215,
439 "params": 45857,
440 "last_loss": 0.0006174481904599816
441 },
442 {
443 "seed": 1,
444 "metric": 0.0004945265245623887,
445 "params": 45857,
446 "last_loss": 0.0004121658857911825
447 },
448 {
449 "seed": 2,
450 "metric": 0.00023093193885870278,
451 "params": 45857,
452 "last_loss": 0.0002016581519274041
453 },
454 {
455 "seed": 3,
456 "metric": 0.0005860979435965419,
457 "params": 45857,
458 "last_loss": 0.00034360544756054876
459 },
460 {
461 "seed": 4,
462 "metric": 0.00023517308000009507,
463 "params": 45857,
464 "last_loss": 0.00022848578926641495
465 },
466 {
467 "seed": 5,
468 "metric": 0.00020430154108908027,
469 "params": 45857,
470 "last_loss": 0.00022899049799889325
471 },
472 {
473 "seed": 6,
474 "metric": 0.000541141489520669,
475 "params": 45857,
476 "last_loss": 0.0004576991230715066
477 },
478 {
479 "seed": 7,
480 "metric": 0.0005798843340016901,
481 "params": 45857,
482 "last_loss": 0.00044673100113868714
483 }
484 ]
485 },
486 {
487 "config": {
488 "lr": 0.006,
489 "weight_decay": 0.0
490 },
491 "per_seed": [
492 0.00012686560512520373,
493 0.0004387780209071934,
494 0.0002368502609897405,
495 0.00015413027722388506,
496 0.00017462033429183066,
497 0.00011361205542925745,
498 0.0002228845260106027,
499 0.0003435298858676106
500 ],
501 "mean": 0.0002264088707306655,
502 "std": 0.00010575777904307542,
503 "n": 8,
504 "details": [
505 {
506 "seed": 0,
507 "metric": 0.00012686560512520373,
508 "params": 45857,
509 "last_loss": 0.00018608432030305265
510 },
511 {
512 "seed": 1,
513 "metric": 0.0004387780209071934,
514 "params": 45857,
515 "last_loss": 0.00024508837144821883
516 },
517 {
518 "seed": 2,
519 "metric": 0.0002368502609897405,
520 "params": 45857,
521 "last_loss": 0.0001869376830291003
522 },
523 {
524 "seed": 3,
525 "metric": 0.00015413027722388506,
526 "params": 45857,
527 "last_loss": 0.00015381897304905578
528 },
529 {
530 "seed": 4,
531 "metric": 0.00017462033429183066,
532 "params": 45857,
533 "last_loss": 0.00021269463730277493
534 },
535 {
536 "seed": 5,
537 "metric": 0.00011361205542925745,
538 "params": 45857,
539 "last_loss": 0.0001970260846428573
540 },
541 {
542 "seed": 6,
543 "metric": 0.0002228845260106027,
544 "params": 45857,
545 "last_loss": 0.00033824800048023464
546 },
547 {
548 "seed": 7,
549 "metric": 0.0003435298858676106,
550 "params": 45857,
551 "last_loss": 0.0003234831592999399
552 }
553 ]
554 },
555 {
556 "config": {
557 "lr": 0.003,
558 "weight_decay": 0.0001
559 },
560 "per_seed": [
561 0.0007010238477960229,
562 0.0004941863589920104,
563 0.0002418020012555644,
564 0.0006359018152579665,
565 0.0002609499788377434,
566 0.0002295856538694352,
567 0.0005036439397372305,
568 0.0005770205752924085
569 ],
570 "mean": 0.0004555142713797977,
571 "std": 0.0001753433538338588,
572 "n": 8,
573 "details": [
574 {
575 "seed": 0,
576 "metric": 0.0007010238477960229,
577 "params": 45857,
578 "last_loss": 0.0006500197574496269
579 },
580 {
581 "seed": 1,
582 "metric": 0.0004941863589920104,
583 "params": 45857,
584 "last_loss": 0.0004117381514515728
585 },
586 {
587 "seed": 2,
588 "metric": 0.0002418020012555644,
589 "params": 45857,
590 "last_loss": 0.00021009275340475142
591 },
592 {
593 "seed": 3,
594 "metric": 0.0006359018152579665,
595 "params": 45857,
596 "last_loss": 0.00037261329242028297
597 },
598 {
599 "seed": 4,
600 "metric": 0.0002609499788377434,
601 "params": 45857,
602 "last_loss": 0.00026486166985705494
603 },
604 {
605 "seed": 5,
606 "metric": 0.0002295856538694352,
607 "params": 45857,
608 "last_loss": 0.0002658980851992965
609 },
610 {
611 "seed": 6,
612 "metric": 0.0005036439397372305,
613 "params": 45857,
614 "last_loss": 0.00045993857202120125
615 },
616 {
617 "seed": 7,
618 "metric": 0.0005770205752924085,
619 "params": 45857,
620 "last_loss": 0.00045114275999367235
621 }
622 ]
623 }
624 ],
625 "parameter_counts": {
626 "baseline": 13313,
627 "idea": 45857
628 }
629}