Persistent Hamiltonian categorical sampler / bench_report.json
Failed on benchmark
1{
2 "bench_version": 1,
3 "track": "categorical_persistent_sequences",
4 "model": "mlp_token_logits",
5 "metric_direction": "lower is better",
6 "n_seeds": 8,
7 "baseline": {
8 "best_cfg": {
9 "lr": 0.003,
10 "temp": 0.8
11 },
12 "sweep": [
13 {
14 "cfg": {
15 "lr": 0.001,
16 "temp": 0.8
17 },
18 "mean": 0.49406249999999996
19 },
20 {
21 "cfg": {
22 "lr": 0.001,
23 "temp": 1.0
24 },
25 "mean": 0.49406249999999996
26 },
27 {
28 "cfg": {
29 "lr": 0.001,
30 "temp": 1.2
31 },
32 "mean": 0.49406249999999996
33 },
34 {
35 "cfg": {
36 "lr": 0.003,
37 "temp": 0.8
38 },
39 "mean": 0.490625
40 },
41 {
42 "cfg": {
43 "lr": 0.003,
44 "temp": 1.0
45 },
46 "mean": 0.490625
47 },
48 {
49 "cfg": {
50 "lr": 0.003,
51 "temp": 1.2
52 },
53 "mean": 0.490625
54 },
55 {
56 "cfg": {
57 "lr": 0.01,
58 "temp": 0.8
59 },
60 "mean": 0.49687499999999996
61 },
62 {
63 "cfg": {
64 "lr": 0.01,
65 "temp": 1.0
66 },
67 "mean": 0.49687499999999996
68 },
69 {
70 "cfg": {
71 "lr": 0.01,
72 "temp": 1.2
73 },
74 "mean": 0.49687499999999996
75 }
76 ],
77 "full": {
78 "mean": 0.49445312500000005,
79 "std": 0.015781830125475786,
80 "per_seed": [
81 0.460625,
82 0.498125,
83 0.514375,
84 0.489375,
85 0.483125,
86 0.50875,
87 0.501875,
88 0.499375
89 ],
90 "n": 8
91 }
92 },
93 "idea": {
94 "mean": 0.48640625000000004,
95 "std": 0.013064406260427612,
96 "per_seed": [
97 0.480625,
98 0.503125,
99 0.48625,
100 0.49875,
101 0.47375,
102 0.464375,
103 0.4825,
104 0.501875
105 ],
106 "n": 8
107 },
108 "comparison": {
109 "delta_mean": -0.008046875000000002,
110 "idea_wins": 4,
111 "n_pairs": 8,
112 "per_seed_diffs": [
113 0.020000000000000018,
114 0.00500000000000006,
115 -0.02812500000000001,
116 0.009375000000000022,
117 -0.009375000000000022,
118 -0.04437500000000005,
119 -0.019374999999999976,
120 0.0024999999999999467
121 ],
122 "p_value": 0.3411,
123 "mde": 0.01794619570947467,
124 "mde_rel_pct": 3.629503951355282,
125 "verdict": "no significant win",
126 "system_worked": false
127 },
128 "mechanism_signature": {
129 "custom_track": {
130 "name": "categorical_persistent_sequences",
131 "file": "categorical_track.py",
132 "domain": "diffusion-sampling"
133 },
134 "idea_config": {
135 "lr": 0.01,
136 "temp": 1.0,
137 "rho": 0.0
138 },
139 "idea_sweep": [
140 {
141 "cfg": {
142 "lr": 0.001,
143 "temp": 0.8,
144 "rho": 0.0
145 },
146 "result": {
147 "mean": 0.4915625,
148 "std": 0.007335434121440939,
149 "per_seed": [
150 0.49375,
151 0.495,
152 0.488125,
153 0.475,
154 0.49,
155 0.495625,
156 0.501875,
157 0.493125
158 ],
159 "n": 8
160 }
161 },
162 {
163 "cfg": {
164 "lr": 0.001,
165 "temp": 0.8,
166 "rho": 0.5
167 },
168 "result": {
169 "mean": 0.49539062500000003,
170 "std": 0.012062408921288268,
171 "per_seed": [
172 0.49375,
173 0.523125,
174 0.493125,
175 0.49375,
176 0.4775,
177 0.49875,
178 0.495,
179 0.488125
180 ],
181 "n": 8
182 }
183 },
184 {
185 "cfg": {
186 "lr": 0.001,
187 "temp": 0.8,
188 "rho": 0.75
189 },
190 "result": {
191 "mean": 0.495390625,
192 "std": 0.011030574961640704,
193 "per_seed": [
194 0.49125,
195 0.515625,
196 0.493125,
197 0.498125,
198 0.47875,
199 0.5075,
200 0.485,
201 0.49375
202 ],
203 "n": 8
204 }
205 },
206 {
207 "cfg": {
208 "lr": 0.001,
209 "temp": 1.0,
210 "rho": 0.0
211 },
212 "result": {
213 "mean": 0.49109375,
214 "std": 0.007842131307080995,
215 "per_seed": [
216 0.493125,
217 0.495,
218 0.48625,
219 0.473125,
220 0.49125,
221 0.496875,
222 0.500625,
223 0.4925
224 ],
225 "n": 8
226 }
227 },
228 {
229 "cfg": {
230 "lr": 0.001,
231 "temp": 1.0,
232 "rho": 0.5
233 },
234 "result": {
235 "mean": 0.495859375,
236 "std": 0.012342513783033639,
237 "per_seed": [
238 0.4925,
239 0.52375,
240 0.49375,
241 0.495,
242 0.47625,
243 0.499375,
244 0.495625,
245 0.490625
246 ],
247 "n": 8
248 }
249 },
250 {
251 "cfg": {
252 "lr": 0.001,
253 "temp": 1.0,
254 "rho": 0.75
255 },
256 "result": {
257 "mean": 0.49484375,
258 "std": 0.011214135541248827,
259 "per_seed": [
260 0.491875,
261 0.515625,
262 0.493125,
263 0.49125,
264 0.47875,
265 0.50875,
266 0.485,
267 0.494375
268 ],
269 "n": 8
270 }
271 },
272 {
273 "cfg": {
274 "lr": 0.001,
275 "temp": 1.2,
276 "rho": 0.0
277 },
278 "result": {
279 "mean": 0.49007812500000003,
280 "std": 0.007351641371787327,
281 "per_seed": [
282 0.49125,
283 0.49625,
284 0.48625,
285 0.474375,
286 0.490625,
287 0.49375,
288 0.500625,
289 0.4875
290 ],
291 "n": 8
292 }
293 },
294 {
295 "cfg": {
296 "lr": 0.001,
297 "temp": 1.2,
298 "rho": 0.5
299 },
300 "result": {
301 "mean": 0.49554687499999994,
302 "std": 0.011727861041527343,
303 "per_seed": [
304 0.491875,
305 0.5225,
306 0.49375,
307 0.493125,
308 0.4775,
309 0.498125,
310 0.49625,
311 0.49125
312 ],
313 "n": 8
314 }
315 },
316 {
317 "cfg": {
318 "lr": 0.001,
319 "temp": 1.2,
320 "rho": 0.75
321 },
322 "result": {
323 "mean": 0.49484375,
324 "std": 0.01113548487662301,
325 "per_seed": [
326 0.49125,
327 0.515625,
328 0.495625,
329 0.491875,
330 0.478125,
331 0.5075,
332 0.485,
333 0.49375
334 ],
335 "n": 8
336 }
337 },
338 {
339 "cfg": {
340 "lr": 0.003,
341 "temp": 0.8,
342 "rho": 0.0
343 },
344 "result": {
345 "mean": 0.48953125,
346 "std": 0.005587985856952403,
347 "per_seed": [
348 0.491875,
349 0.4975,
350 0.49125,
351 0.486875,
352 0.4775,
353 0.490625,
354 0.486875,
355 0.49375
356 ],
357 "n": 8
358 }
359 },
360 {
361 "cfg": {
362 "lr": 0.003,
363 "temp": 0.8,
364 "rho": 0.5
365 },
366 "result": {
367 "mean": 0.49203125000000003,
368 "std": 0.013609904194648118,
369 "per_seed": [
370 0.503125,
371 0.51375,
372 0.48375,
373 0.496875,
374 0.464375,
375 0.495625,
376 0.490625,
377 0.488125
378 ],
379 "n": 8
380 }
381 },
382 {
383 "cfg": {
384 "lr": 0.003,
385 "temp": 0.8,
386 "rho": 0.75
387 },
388 "result": {
389 "mean": 0.494921875,
390 "std": 0.00963411449534284,
391 "per_seed": [
392 0.488125,
393 0.514375,
394 0.49,
395 0.49375,
396 0.483125,
397 0.505625,
398 0.48875,
399 0.495625
400 ],
401 "n": 8
402 }
403 },
404 {
405 "cfg": {
406 "lr": 0.003,
407 "temp": 1.0,
408 "rho": 0.0
409 },
410 "result": {
411 "mean": 0.488984375,
412 "std": 0.0071393598441579515,
413 "per_seed": [
414 0.494375,
415 0.496875,
416 0.49,
417 0.484375,
418 0.475625,
419 0.486875,
420 0.485,
421 0.49875
422 ],
423 "n": 8
424 }
425 },
426 {
427 "cfg": {
428 "lr": 0.003,
429 "temp": 1.0,
430 "rho": 0.5
431 },
432 "result": {
433 "mean": 0.491328125,
434 "std": 0.013193413588202828,
435 "per_seed": [
436 0.5025,
437 0.513125,
438 0.481875,
439 0.494375,
440 0.465,
441 0.491875,
442 0.49125,
443 0.490625
444 ],
445 "n": 8
446 }
447 },
448 {
449 "cfg": {
450 "lr": 0.003,
451 "temp": 1.0,
452 "rho": 0.75
453 },
454 "result": {
455 "mean": 0.49421875,
456 "std": 0.009456675469608747,
457 "per_seed": [
458 0.489375,
459 0.510625,
460 0.485,
461 0.495,
462 0.483125,
463 0.5075,
464 0.488125,
465 0.495
466 ],
467 "n": 8
468 }
469 },
470 {
471 "cfg": {
472 "lr": 0.003,
473 "temp": 1.2,
474 "rho": 0.0
475 },
476 "result": {
477 "mean": 0.490234375,
478 "std": 0.0072074282156935204,
479 "per_seed": [
480 0.495625,
481 0.5,
482 0.490625,
483 0.486875,
484 0.475,
485 0.490625,
486 0.486875,
487 0.49625
488 ],
489 "n": 8
490 }
491 },
492 {
493 "cfg": {
494 "lr": 0.003,
495 "temp": 1.2,
496 "rho": 0.5
497 },
498 "result": {
499 "mean": 0.49203125000000003,
500 "std": 0.013049447725382855,
501 "per_seed": [
502 0.50125,
503 0.5125,
504 0.48375,
505 0.495625,
506 0.464375,
507 0.493125,
508 0.495,
509 0.490625
510 ],
511 "n": 8
512 }
513 },
514 {
515 "cfg": {
516 "lr": 0.003,
517 "temp": 1.2,
518 "rho": 0.75
519 },
520 "result": {
521 "mean": 0.494609375,
522 "std": 0.010448616366982527,
523 "per_seed": [
524 0.494375,
525 0.51125,
526 0.4825,
527 0.495625,
528 0.48,
529 0.50875,
530 0.48875,
531 0.495625
532 ],
533 "n": 8
534 }
535 },
536 {
537 "cfg": {
538 "lr": 0.01,
539 "temp": 0.8,
540 "rho": 0.0
541 },
542 "result": {
543 "mean": 0.487890625,
544 "std": 0.01414882359012138,
545 "per_seed": [
546 0.48,
547 0.51,
548 0.488125,
549 0.503125,
550 0.4725,
551 0.468125,
552 0.48125,
553 0.5
554 ],
555 "n": 8
556 }
557 },
558 {
559 "cfg": {
560 "lr": 0.01,
561 "temp": 0.8,
562 "rho": 0.5
563 },
564 "result": {
565 "mean": 0.494140625,
566 "std": 0.014337374331947086,
567 "per_seed": [
568 0.49625,
569 0.525625,
570 0.485,
571 0.495,
572 0.47625,
573 0.48375,
574 0.4875,
575 0.50375
576 ],
577 "n": 8
578 }
579 },
580 {
581 "cfg": {
582 "lr": 0.01,
583 "temp": 0.8,
584 "rho": 0.75
585 },
586 "result": {
587 "mean": 0.494375,
588 "std": 0.012480453467322404,
589 "per_seed": [
590 0.48,
591 0.5225,
592 0.485,
593 0.49875,
594 0.490625,
595 0.490625,
596 0.48625,
597 0.50125
598 ],
599 "n": 8
600 }
601 },
602 {
603 "cfg": {
604 "lr": 0.01,
605 "temp": 1.0,
606 "rho": 0.0
607 },
608 "result": {
609 "mean": 0.48640625000000004,
610 "std": 0.013064406260427612,
611 "per_seed": [
612 0.480625,
613 0.503125,
614 0.48625,
615 0.49875,
616 0.47375,
617 0.464375,
618 0.4825,
619 0.501875
620 ],
621 "n": 8
622 }
623 },
624 {
625 "cfg": {
626 "lr": 0.01,
627 "temp": 1.0,
628 "rho": 0.5
629 },
630 "result": {
631 "mean": 0.4946875,
632 "std": 0.012751378601939463,
633 "per_seed": [
634 0.495625,
635 0.518125,
636 0.48625,
637 0.498125,
638 0.475625,
639 0.48625,
640 0.48875,
641 0.50875
642 ],
643 "n": 8
644 }
645 },
646 {
647 "cfg": {
648 "lr": 0.01,
649 "temp": 1.0,
650 "rho": 0.75
651 },
652 "result": {
653 "mean": 0.495,
654 "std": 0.010798220339481851,
655 "per_seed": [
656 0.481875,
657 0.518125,
658 0.48375,
659 0.4975,
660 0.49375,
661 0.493125,
662 0.489375,
663 0.5025
664 ],
665 "n": 8
666 }
667 },
668 {
669 "cfg": {
670 "lr": 0.01,
671 "temp": 1.2,
672 "rho": 0.0
673 },
674 "result": {
675 "mean": 0.48726562500000004,
676 "std": 0.012251185768503196,
677 "per_seed": [
678 0.483125,
679 0.499375,
680 0.489375,
681 0.495625,
682 0.47375,
683 0.464375,
684 0.489375,
685 0.503125
686 ],
687 "n": 8
688 }
689 },
690 {
691 "cfg": {
692 "lr": 0.01,
693 "temp": 1.2,
694 "rho": 0.5
695 },
696 "result": {
697 "mean": 0.49382812499999995,
698 "std": 0.013329645282578786,
699 "per_seed": [
700 0.49625,
701 0.516875,
702 0.485,
703 0.49375,
704 0.47375,
705 0.485625,
706 0.488125,
707 0.51125
708 ],
709 "n": 8
710 }
711 },
712 {
713 "cfg": {
714 "lr": 0.01,
715 "temp": 1.2,
716 "rho": 0.75
717 },
718 "result": {
719 "mean": 0.49671875,
720 "std": 0.010325512623473003,
721 "per_seed": [
722 0.4825,
723 0.51875,
724 0.486875,
725 0.495,
726 0.49625,
727 0.494375,
728 0.495625,
729 0.504375
730 ],
731 "n": 8
732 }
733 }
734 ],
735 "mechanism_signature": {
736 "prediction": "Gaussian refresh sign correlation is 2 asin(rho)/pi",
737 "rho": 0.75,
738 "rows": [
739 {
740 "predicted_sign_corr": 0.5398930876747683,
741 "observed_sign_corr": 0.060386206896551727
742 },
743 {
744 "predicted_sign_corr": 0.5398930876747683,
745 "observed_sign_corr": 0.06482068965517242
746 },
747 {
748 "predicted_sign_corr": 0.5398930876747683,
749 "observed_sign_corr": 0.07049655172413793
750 },
751 {
752 "predicted_sign_corr": 0.5398930876747683,
753 "observed_sign_corr": 0.06498448275862069
754 }
755 ],
756 "mean_abs_error": 0.4747211049161476,
757 "tolerance": 0.04,
758 "confirmed": false
759 }
760 }
761}