1{
  2  "bench_version": 1,
  3  "track": "correlated_token_moe_regression",
  4  "model": "custom_moe",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "best_cfg": {
  9      "lr": 0.006,
 10      "aux": 0.01,
 11      "balanced": false,
 12      "K": 4
 13    },
 14    "sweep": [
 15      {
 16        "cfg": {
 17          "lr": 0.0015,
 18          "aux": 0.0,
 19          "balanced": false,
 20          "K": 4
 21        },
 22        "mean": 0.0866615679115057,
 23        "per_seed": [
 24          0.08718974888324738,
 25          0.09299269318580627,
 26          0.08348067104816437,
 27          0.08298315852880478
 28        ]
 29      },
 30      {
 31        "cfg": {
 32          "lr": 0.0015,
 33          "aux": 0.01,
 34          "balanced": false,
 35          "K": 4
 36        },
 37        "mean": 0.08026662468910217,
 38        "per_seed": [
 39          0.06962176412343979,
 40          0.09358783811330795,
 41          0.08609369397163391,
 42          0.07176320254802704
 43        ]
 44      },
 45      {
 46        "cfg": {
 47          "lr": 0.0015,
 48          "aux": 0.1,
 49          "balanced": false,
 50          "K": 4
 51        },
 52        "mean": 0.0802982971072197,
 53        "per_seed": [
 54          0.06983038038015366,
 55          0.09358862042427063,
 56          0.08610628545284271,
 57          0.07166790217161179
 58        ]
 59      },
 60      {
 61        "cfg": {
 62          "lr": 0.003,
 63          "aux": 0.0,
 64          "balanced": false,
 65          "K": 4
 66        },
 67        "mean": 0.07278427481651306,
 68        "per_seed": [
 69          0.07727840542793274,
 70          0.07310336083173752,
 71          0.07483050227165222,
 72          0.06592483073472977
 73        ]
 74      },
 75      {
 76        "cfg": {
 77          "lr": 0.003,
 78          "aux": 0.01,
 79          "balanced": false,
 80          "K": 4
 81        },
 82        "mean": 0.06707096565514803,
 83        "per_seed": [
 84          0.06202782317996025,
 85          0.07177857309579849,
 86          0.07393620163202286,
 87          0.060541264712810516
 88        ]
 89      },
 90      {
 91        "cfg": {
 92          "lr": 0.003,
 93          "aux": 0.1,
 94          "balanced": false,
 95          "K": 4
 96        },
 97        "mean": 0.06711035128682852,
 98        "per_seed": [
 99          0.062259428203105927,
100          0.07183852046728134,
101          0.0737687274813652,
102          0.0605747289955616
103        ]
104      },
105      {
106        "cfg": {
107          "lr": 0.006,
108          "aux": 0.0,
109          "balanced": false,
110          "K": 4
111        },
112        "mean": 0.05864693410694599,
113        "per_seed": [
114          0.062149204313755035,
115          0.05799993872642517,
116          0.06060343608260155,
117          0.05383515730500221
118        ]
119      },
120      {
121        "cfg": {
122          "lr": 0.006,
123          "aux": 0.01,
124          "balanced": false,
125          "K": 4
126        },
127        "mean": 0.056222427636384964,
128        "per_seed": [
129          0.049452465027570724,
130          0.058252543210983276,
131          0.05263408645987511,
132          0.06455061584711075
133        ]
134      },
135      {
136        "cfg": {
137          "lr": 0.006,
138          "aux": 0.1,
139          "balanced": false,
140          "K": 4
141        },
142        "mean": 0.0565244285389781,
143        "per_seed": [
144          0.04950065538287163,
145          0.0584375374019146,
146          0.052471455186605453,
147          0.06568806618452072
148        ]
149      }
150    ],
151    "full": {
152      "per_seed": [
153        0.049452465027570724,
154        0.058252543210983276,
155        0.05263408645987511,
156        0.06455061584711075,
157        0.053144052624702454,
158        0.07924187928438187,
159        0.04448141157627106,
160        0.05137491226196289
161      ],
162      "mean": 0.056641495786607265,
163      "records": [
164        {
165          "seed": 0,
166          "metric": 0.049452465027570724,
167          "routing": {
168            "load_cv": 0.0913184674641444,
169            "loads": [
170              346.0,
171              426.0,
172              354.0,
173              395.0,
174              422.0,
175              452.0,
176              373.0,
177              432.0
178            ],
179            "max_shell_count_spread": 311,
180            "reassigned_fraction": 0.0,
181            "shells": 4
182          }
183        },
184        {
185          "seed": 1,
186          "metric": 0.058252543210983276,
187          "routing": {
188            "load_cv": 0.13504628836069504,
189            "loads": [
190              382.0,
191              403.0,
192              373.0,
193              531.0,
194              386.0,
195              400.0,
196              395.0,
197              330.0
198            ],
199            "max_shell_count_spread": 184,
200            "reassigned_fraction": 0.0,
201            "shells": 4
202          }
203        },
204        {
205          "seed": 2,
206          "metric": 0.05263408645987511,
207          "routing": {
208            "load_cv": 0.12644168616401794,
209            "loads": [
210              397.0,
211              428.0,
212              348.0,
213              306.0,
214              433.0,
215              424.0,
216              385.0,
217              479.0
218            ],
219            "max_shell_count_spread": 159,
220            "reassigned_fraction": 0.0,
221            "shells": 4
222          }
223        },
224        {
225          "seed": 3,
226          "metric": 0.06455061584711075,
227          "routing": {
228            "load_cv": 0.06754628043053149,
229            "loads": [
230              398.0,
231              374.0,
232              455.0,
233              379.0,
234              394.0,
235              387.0,
236              433.0,
237              380.0
238            ],
239            "max_shell_count_spread": 269,
240            "reassigned_fraction": 0.0,
241            "shells": 4
242          }
243        },
244        {
245          "seed": 4,
246          "metric": 0.053144052624702454,
247          "routing": {
248            "load_cv": 0.1465595015684756,
249            "loads": [
250              412.0,
251              382.0,
252              296.0,
253              394.0,
254              490.0,
255              373.0,
256              372.0,
257              481.0
258            ],
259            "max_shell_count_spread": 297,
260            "reassigned_fraction": 0.0,
261            "shells": 4
262          }
263        },
264        {
265          "seed": 5,
266          "metric": 0.07924187928438187,
267          "routing": {
268            "load_cv": 0.17972200755611428,
269            "loads": [
270              437.0,
271              391.0,
272              372.0,
273              424.0,
274              316.0,
275              562.0,
276              353.0,
277              345.0
278            ],
279            "max_shell_count_spread": 321,
280            "reassigned_fraction": 0.0,
281            "shells": 4
282          }
283        },
284        {
285          "seed": 6,
286          "metric": 0.04448141157627106,
287          "routing": {
288            "load_cv": 0.08075812652606547,
289            "loads": [
290              360.0,
291              407.0,
292              417.0,
293              424.0,
294              348.0,
295              376.0,
296              445.0,
297              423.0
298            ],
299            "max_shell_count_spread": 264,
300            "reassigned_fraction": 0.0,
301            "shells": 4
302          }
303        },
304        {
305          "seed": 7,
306          "metric": 0.05137491226196289,
307          "routing": {
308            "load_cv": 0.10753633107001559,
309            "loads": [
310              341.0,
311              418.0,
312              375.0,
313              380.0,
314              347.0,
315              427.0,
316              465.0,
317              447.0
318            ],
319            "max_shell_count_spread": 262,
320            "reassigned_fraction": 0.0,
321            "shells": 4
322          }
323        }
324      ],
325      "cfg": {
326        "lr": 0.006,
327        "aux": 0.01,
328        "balanced": false,
329        "K": 4
330      }
331    }
332  },
333  "idea": {
334    "per_seed": [
335      0.05467335134744644,
336      0.05477738380432129,
337      0.056872498244047165,
338      0.051517780870199203,
339      0.05167350545525551,
340      0.06755945831537247,
341      0.04719924181699753,
342      0.04966289922595024
343    ],
344    "mean": 0.05424201488494873,
345    "records": [
346      {
347        "seed": 0,
348        "metric": 0.05467335134744644,
349        "routing": {
350          "load_cv": 0.0,
351          "loads": [
352            400.0,
353            400.0,
354            400.0,
355            400.0,
356            400.0,
357            400.0,
358            400.0,
359            400.0
360          ],
361          "max_shell_count_spread": 0,
362          "reassigned_fraction": 0.61625,
363          "shells": 8
364        }
365      },
366      {
367        "seed": 1,
368        "metric": 0.05477738380432129,
369        "routing": {
370          "load_cv": 0.0,
371          "loads": [
372            400.0,
373            400.0,
374            400.0,
375            400.0,
376            400.0,
377            400.0,
378            400.0,
379            400.0
380          ],
381          "max_shell_count_spread": 0,
382          "reassigned_fraction": 0.3775,
383          "shells": 8
384        }
385      },
386      {
387        "seed": 2,
388        "metric": 0.056872498244047165,
389        "routing": {
390          "load_cv": 0.0,
391          "loads": [
392            400.0,
393            400.0,
394            400.0,
395            400.0,
396            400.0,
397            400.0,
398            400.0,
399            400.0
400          ],
401          "max_shell_count_spread": 0,
402          "reassigned_fraction": 0.5353125,
403          "shells": 8
404        }
405      },
406      {
407        "seed": 3,
408        "metric": 0.051517780870199203,
409        "routing": {
410          "load_cv": 0.0,
411          "loads": [
412            400.0,
413            400.0,
414            400.0,
415            400.0,
416            400.0,
417            400.0,
418            400.0,
419            400.0
420          ],
421          "max_shell_count_spread": 0,
422          "reassigned_fraction": 0.5540625,
423          "shells": 8
424        }
425      },
426      {
427        "seed": 4,
428        "metric": 0.05167350545525551,
429        "routing": {
430          "load_cv": 0.0,
431          "loads": [
432            400.0,
433            400.0,
434            400.0,
435            400.0,
436            400.0,
437            400.0,
438            400.0,
439            400.0
440          ],
441          "max_shell_count_spread": 0,
442          "reassigned_fraction": 0.4,
443          "shells": 8
444        }
445      },
446      {
447        "seed": 5,
448        "metric": 0.06755945831537247,
449        "routing": {
450          "load_cv": 0.0,
451          "loads": [
452            400.0,
453            400.0,
454            400.0,
455            400.0,
456            400.0,
457            400.0,
458            400.0,
459            400.0
460          ],
461          "max_shell_count_spread": 0,
462          "reassigned_fraction": 0.5334375,
463          "shells": 8
464        }
465      },
466      {
467        "seed": 6,
468        "metric": 0.04719924181699753,
469        "routing": {
470          "load_cv": 0.0,
471          "loads": [
472            400.0,
473            400.0,
474            400.0,
475            400.0,
476            400.0,
477            400.0,
478            400.0,
479            400.0
480          ],
481          "max_shell_count_spread": 0,
482          "reassigned_fraction": 0.534375,
483          "shells": 8
484        }
485      },
486      {
487        "seed": 7,
488        "metric": 0.04966289922595024,
489        "routing": {
490          "load_cv": 0.0,
491          "loads": [
492            400.0,
493            400.0,
494            400.0,
495            400.0,
496            400.0,
497            400.0,
498            400.0,
499            400.0
500          ],
501          "max_shell_count_spread": 0,
502          "reassigned_fraction": 0.4796875,
503          "shells": 8
504        }
505      }
506    ],
507    "cfg": {
508      "lr": 0.006,
509      "balanced": true,
510      "K": 8
511    }
512  },
513  "comparison": {
514    "delta_mean": -0.002399480901658535,
515    "idea_wins": 5,
516    "n_pairs": 8,
517    "per_seed_diffs": [
518      0.005220886319875717,
519      -0.0034751594066619873,
520      0.004238411784172058,
521      -0.013032834976911545,
522      -0.0014705471694469452,
523      -0.0116824209690094,
524      0.002717830240726471,
525      -0.0017120130360126495
526    ],
527    "p_value": 0.38035,
528    "mde": 0.00574154155483705,
529    "mde_rel_pct": 10.136634767676144,
530    "verdict": "no significant win",
531    "system_worked": false
532  },
533  "mechanism_signature": {
534    "custom_track": {
535      "name": "correlated_token_moe_regression",
536      "file": "shell_moe_bench_track.py",
537      "domain": "moe-routing"
538    },
539    "prediction": "per-shell expert count spread <= 1",
540    "baseline_mean_load_cv": 0.11686608614250746,
541    "idea_mean_load_cv": 0.0,
542    "baseline_mean_shell_spread": 258.375,
543    "idea_mean_shell_spread": 0.0,
544    "idea_mean_reassigned_fraction": 0.5038281250000001,
545    "confirmed": true
546  },
547  "idea_trials": [
548    {
549      "cfg": {
550        "lr": 0.0015,
551        "balanced": true,
552        "K": 2
553      },
554      "mean": 0.08199466113001108,
555      "per_seed": [
556        0.07786157727241516,
557        0.0871136337518692,
558        0.08436500281095505,
559        0.07892747968435287,
560        0.07485600560903549,
561        0.10391078889369965,
562        0.07056240737438202,
563        0.07836039364337921
564      ]
565    },
566    {
567      "cfg": {
568        "lr": 0.0015,
569        "balanced": true,
570        "K": 4
571      },
572      "mean": 0.08325883187353611,
573      "per_seed": [
574        0.07747699320316315,
575        0.09112893790006638,
576        0.08618856221437454,
577        0.07941422611474991,
578        0.07391377538442612,
579        0.10588844120502472,
580        0.07102843374013901,
581        0.08103128522634506
582      ]
583    },
584    {
585      "cfg": {
586        "lr": 0.0015,
587        "balanced": true,
588        "K": 8
589      },
590      "mean": 0.08230428863316774,
591      "per_seed": [
592        0.07840844988822937,
593        0.08882392942905426,
594        0.08409003913402557,
595        0.08046981692314148,
596        0.07284295558929443,
597        0.10753589123487473,
598        0.06986328959465027,
599        0.07639993727207184
600      ]
601    },
602    {
603      "cfg": {
604        "lr": 0.003,
605        "balanced": true,
606        "K": 2
607      },
608      "mean": 0.06749503640457988,
609      "per_seed": [
610        0.06871458142995834,
611        0.06850986182689667,
612        0.07267368584871292,
613        0.06359779834747314,
614        0.06259164959192276,
615        0.08258448541164398,
616        0.05686591938138008,
617        0.06442230939865112
618      ]
619    },
620    {
621      "cfg": {
622        "lr": 0.003,
623        "balanced": true,
624        "K": 4
625      },
626      "mean": 0.06823915103450418,
627      "per_seed": [
628        0.0692436695098877,
629        0.07023227959871292,
630        0.07335231453180313,
631        0.061882417649030685,
632        0.06339849531650543,
633        0.08482187241315842,
634        0.057540178298950195,
635        0.06544198095798492
636      ]
637    },
638    {
639      "cfg": {
640        "lr": 0.003,
641        "balanced": true,
642        "K": 8
643      },
644      "mean": 0.06749762082472444,
645      "per_seed": [
646        0.06905151903629303,
647        0.06874477118253708,
648        0.07229793071746826,
649        0.06277107447385788,
650        0.061685409396886826,
651        0.08680634945631027,
652        0.05603644251823425,
653        0.06258746981620789
654      ]
655    },
656    {
657      "cfg": {
658        "lr": 0.006,
659        "balanced": true,
660        "K": 2
661      },
662      "mean": 0.054870990104973316,
663      "per_seed": [
664        0.05411909893155098,
665        0.057356927543878555,
666        0.05673087760806084,
667        0.055093344300985336,
668        0.051891688257455826,
669        0.06572796404361725,
670        0.04767368361353874,
671        0.050374336540699005
672      ]
673    },
674    {
675      "cfg": {
676        "lr": 0.006,
677        "balanced": true,
678        "K": 4
679      },
680      "mean": 0.05486724339425564,
681      "per_seed": [
682        0.05602085962891579,
683        0.056902941316366196,
684        0.05770782381296158,
685        0.052157238125801086,
686        0.054022110998630524,
687        0.06586991250514984,
688        0.046959299594163895,
689        0.0492977611720562
690      ]
691    },
692    {
693      "cfg": {
694        "lr": 0.006,
695        "balanced": true,
696        "K": 8
697      },
698      "mean": 0.05424201488494873,
699      "per_seed": [
700        0.05467335134744644,
701        0.05477738380432129,
702        0.056872498244047165,
703        0.051517780870199203,
704        0.05167350545525551,
705        0.06755945831537247,
706        0.04719924181699753,
707        0.04966289922595024
708      ]
709    }
710  ]
711}