1{
  2  "bench_version": 1,
  3  "track": "heterogeneous_regime_moe",
  4  "model": "custom_sparse_moe",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "best_cfg": {
  9      "lr": 0.006,
 10      "alpha": 0.0,
 11      "aux": 0.05,
 12      "beta": 0.9,
 13      "wd": 0.0,
 14      "epochs": 3
 15    },
 16    "sweep": [
 17      {
 18        "cfg": {
 19          "lr": 0.001,
 20          "alpha": 0.0,
 21          "aux": 0.0,
 22          "beta": 0.9,
 23          "wd": 0.0,
 24          "epochs": 3
 25        },
 26        "mean": 0.3828819803893566,
 27        "per_seed": [
 28          0.38268065452575684,
 29          0.42182663083076477,
 30          0.39980241656303406,
 31          0.4181539714336395,
 32          0.3709557354450226,
 33          0.3404829204082489,
 34          0.3406737446784973,
 35          0.3884797692298889
 36        ]
 37      },
 38      {
 39        "cfg": {
 40          "lr": 0.001,
 41          "alpha": 0.0,
 42          "aux": 0.02,
 43          "beta": 0.9,
 44          "wd": 0.0,
 45          "epochs": 3
 46        },
 47        "mean": 0.38272077590227127,
 48        "per_seed": [
 49          0.38274234533309937,
 50          0.42027127742767334,
 51          0.40094804763793945,
 52          0.41814735531806946,
 53          0.37042883038520813,
 54          0.34024134278297424,
 55          0.34057343006134033,
 56          0.38841357827186584
 57        ]
 58      },
 59      {
 60        "cfg": {
 61          "lr": 0.001,
 62          "alpha": 0.0,
 63          "aux": 0.05,
 64          "beta": 0.9,
 65          "wd": 0.0,
 66          "epochs": 3
 67        },
 68        "mean": 0.38203955814242363,
 69        "per_seed": [
 70          0.3827840983867645,
 71          0.41726404428482056,
 72          0.4006299376487732,
 73          0.4182635545730591,
 74          0.37063243985176086,
 75          0.3401484489440918,
 76          0.33957916498184204,
 77          0.387014776468277
 78        ]
 79      },
 80      {
 81        "cfg": {
 82          "lr": 0.001,
 83          "alpha": 0.0,
 84          "aux": 0.1,
 85          "beta": 0.9,
 86          "wd": 0.0,
 87          "epochs": 3
 88        },
 89        "mean": 0.3819722458720207,
 90        "per_seed": [
 91          0.3825948238372803,
 92          0.4162457585334778,
 93          0.40236616134643555,
 94          0.4189237058162689,
 95          0.37040382623672485,
 96          0.3401601314544678,
 97          0.33924177289009094,
 98          0.3858417868614197
 99        ]
100      },
101      {
102        "cfg": {
103          "lr": 0.003,
104          "alpha": 0.0,
105          "aux": 0.0,
106          "beta": 0.9,
107          "wd": 0.0,
108          "epochs": 3
109        },
110        "mean": 0.322677806019783,
111        "per_seed": [
112          0.336835652589798,
113          0.3815375566482544,
114          0.31310850381851196,
115          0.36289894580841064,
116          0.2942163646221161,
117          0.27027586102485657,
118          0.2819417417049408,
119          0.34060782194137573
120        ]
121      },
122      {
123        "cfg": {
124          "lr": 0.003,
125          "alpha": 0.0,
126          "aux": 0.02,
127          "beta": 0.9,
128          "wd": 0.0,
129          "epochs": 3
130        },
131        "mean": 0.3223222754895687,
132        "per_seed": [
133          0.33747565746307373,
134          0.37728649377822876,
135          0.3187519311904907,
136          0.3634164333343506,
137          0.28765812516212463,
138          0.26715168356895447,
139          0.28378787636756897,
140          0.3430500030517578
141        ]
142      },
143      {
144        "cfg": {
145          "lr": 0.003,
146          "alpha": 0.0,
147          "aux": 0.05,
148          "beta": 0.9,
149          "wd": 0.0,
150          "epochs": 3
151        },
152        "mean": 0.3225990757346153,
153        "per_seed": [
154          0.3332405090332031,
155          0.37485089898109436,
156          0.3205052614212036,
157          0.36014488339424133,
158          0.2920096516609192,
159          0.2685995101928711,
160          0.28830453753471375,
161          0.34313735365867615
162        ]
163      },
164      {
165        "cfg": {
166          "lr": 0.003,
167          "alpha": 0.0,
168          "aux": 0.1,
169          "beta": 0.9,
170          "wd": 0.0,
171          "epochs": 3
172        },
173        "mean": 0.32372206822037697,
174        "per_seed": [
175          0.3341885805130005,
176          0.374936580657959,
177          0.3235602080821991,
178          0.3638923466205597,
179          0.29552894830703735,
180          0.26977014541625977,
181          0.29074564576148987,
182          0.3371540904045105
183        ]
184      },
185      {
186        "cfg": {
187          "lr": 0.006,
188          "alpha": 0.0,
189          "aux": 0.0,
190          "beta": 0.9,
191          "wd": 0.0,
192          "epochs": 3
193        },
194        "mean": 0.3063250947743654,
195        "per_seed": [
196          0.340360552072525,
197          0.3520633578300476,
198          0.29556065797805786,
199          0.348251074552536,
200          0.26277485489845276,
201          0.275777131319046,
202          0.24570734798908234,
203          0.3301057815551758
204        ]
205      },
206      {
207        "cfg": {
208          "lr": 0.006,
209          "alpha": 0.0,
210          "aux": 0.02,
211          "beta": 0.9,
212          "wd": 0.0,
213          "epochs": 3
214        },
215        "mean": 0.3056254591792822,
216        "per_seed": [
217          0.3370864689350128,
218          0.3551115095615387,
219          0.29941967129707336,
220          0.33801084756851196,
221          0.26092731952667236,
222          0.2828543484210968,
223          0.23862560093402863,
224          0.3329679071903229
225        ]
226      },
227      {
228        "cfg": {
229          "lr": 0.006,
230          "alpha": 0.0,
231          "aux": 0.05,
232          "beta": 0.9,
233          "wd": 0.0,
234          "epochs": 3
235        },
236        "mean": 0.3030446618795395,
237        "per_seed": [
238          0.33402499556541443,
239          0.3469650149345398,
240          0.2999320328235626,
241          0.3352453112602234,
242          0.2639300525188446,
243          0.2693466544151306,
244          0.242201566696167,
245          0.33271166682243347
246        ]
247      },
248      {
249        "cfg": {
250          "lr": 0.006,
251          "alpha": 0.0,
252          "aux": 0.1,
253          "beta": 0.9,
254          "wd": 0.0,
255          "epochs": 3
256        },
257        "mean": 0.304129621013999,
258        "per_seed": [
259          0.33261367678642273,
260          0.34067946672439575,
261          0.2965738773345947,
262          0.3427000343799591,
263          0.2676871120929718,
264          0.26694804430007935,
265          0.2444242388010025,
266          0.3414105176925659
267        ]
268      }
269    ],
270    "full": {
271      "mean": 0.3030446618795395,
272      "std": 0.037387950188241835,
273      "per_seed": [
274        0.33402499556541443,
275        0.3469650149345398,
276        0.2999320328235626,
277        0.3352453112602234,
278        0.2639300525188446,
279        0.2693466544151306,
280        0.242201566696167,
281        0.33271166682243347
282      ],
283      "n": 8,
284      "signatures": [
285        {
286          "load_cv": 0.562938723734806,
287          "raw_cv": 0.16369719803202323,
288          "pressure": [
289            0.09628282487392426,
290            0.081390880048275,
291            0.08156248927116394,
292            0.09052031487226486,
293            0.108507439494133,
294            0.11428291350603104,
295            0.06951107084751129,
296            0.07551244646310806
297          ]
298        },
299        {
300          "load_cv": 0.5744562491404501,
301          "raw_cv": 0.1282355640205095,
302          "pressure": [
303            0.07845945656299591,
304            0.08906298130750656,
305            0.10167022794485092,
306            0.09873484820127487,
307            0.07295134663581848,
308            0.07633225619792938,
309            0.11173825711011887,
310            0.08862103521823883
311          ]
312        },
313        {
314          "load_cv": 0.6500000745006104,
315          "raw_cv": 0.18785403298311348,
316          "pressure": [
317            0.11149176955223083,
318            0.08400898426771164,
319            0.09686370939016342,
320            0.06456371396780014,
321            0.08030611276626587,
322            0.09944741427898407,
323            0.06263477355241776,
324            0.1182539090514183
325          ]
326        },
327        {
328          "load_cv": 0.617089927191566,
329          "raw_cv": 0.15858341208470625,
330          "pressure": [
331            0.07222633063793182,
332            0.06700322031974792,
333            0.09097043424844742,
334            0.0720442607998848,
335            0.1073877140879631,
336            0.09829375892877579,
337            0.0978299006819725,
338            0.11181474477052689
339          ]
340        },
341        {
342          "load_cv": 0.4471017718279356,
343          "raw_cv": 0.1292689442624241,
344          "pressure": [
345            0.09001918137073517,
346            0.10562513768672943,
347            0.0990309864282608,
348            0.06985172629356384,
349            0.09212449193000793,
350            0.09441789984703064,
351            0.09658448398113251,
352            0.06991647183895111
353          ]
354        },
355        {
356          "load_cv": 0.5402776598887137,
357          "raw_cv": 0.15026783943056055,
358          "pressure": [
359            0.0705578625202179,
360            0.10928625613451004,
361            0.08838503062725067,
362            0.09580148756504059,
363            0.10527030378580093,
364            0.06459853053092957,
365            0.11027625948190689,
366            0.07339463382959366
367          ]
368        },
369        {
370          "load_cv": 0.5783597575594265,
371          "raw_cv": 0.11876459419632313,
372          "pressure": [
373            0.0868268609046936,
374            0.09471505880355835,
375            0.11200853437185287,
376            0.08284948021173477,
377            0.08226083219051361,
378            0.08512511849403381,
379            0.09811943769454956,
380            0.07566508650779724
381          ]
382        },
383        {
384          "load_cv": 0.4230839014019498,
385          "raw_cv": 0.09526789188308794,
386          "pressure": [
387            0.08401855081319809,
388            0.09309402853250504,
389            0.08401310443878174,
390            0.08053044974803925,
391            0.0959775522351265,
392            0.10520738363265991,
393            0.09375947713851929,
394            0.08096984028816223
395          ]
396        }
397      ]
398    }
399  },
400  "idea": {
401    "mean": 0.30424891598522663,
402    "std": 0.049616770951017984,
403    "per_seed": [
404      0.34059908986091614,
405      0.3489361107349396,
406      0.2619922161102295,
407      0.3619314134120941,
408      0.27461159229278564,
409      0.2645150423049927,
410      0.22574420273303986,
411      0.35566166043281555
412    ],
413    "n": 8,
414    "signatures": [
415      {
416        "load_cv": 0.3511409759493393,
417        "raw_cv": 0.17809261381483654,
418        "pressure": [
419          0.09716491401195526,
420          0.0805320143699646,
421          0.08110848069190979,
422          0.08782943338155746,
423          0.10998763889074326,
424          0.11640482395887375,
425          0.07011502236127853,
426          0.07442808896303177
427        ]
428      },
429      {
430        "load_cv": 0.5395368337588062,
431        "raw_cv": 0.18354195356222183,
432        "pressure": [
433          0.07514716684818268,
434          0.08768722414970398,
435          0.10187951475381851,
436          0.10252103954553604,
437          0.07107587903738022,
438          0.07342103868722916,
439          0.11874773353338242,
440          0.08709078282117844
441        ]
442      },
443      {
444        "load_cv": 0.5327288508372604,
445        "raw_cv": 0.2340563237648359,
446        "pressure": [
447          0.11677199602127075,
448          0.07979930937290192,
449          0.09878582507371902,
450          0.062348250299692154,
451          0.07951708883047104,
452          0.09967383742332458,
453          0.05981113389134407,
454          0.1208629459142685
455        ]
456      },
457      {
458        "load_cv": 0.48010414838406806,
459        "raw_cv": 0.2041649222357629,
460        "pressure": [
461          0.07007212191820145,
462          0.06405238062143326,
463          0.09175959974527359,
464          0.07002781331539154,
465          0.11198381334543228,
466          0.09871535003185272,
467          0.09743785858154297,
468          0.11352145671844482
469        ]
470      },
471      {
472        "load_cv": 0.2900000214553521,
473        "raw_cv": 0.14455384015921344,
474        "pressure": [
475          0.09033729881048203,
476          0.10671012848615646,
477          0.09973443299531937,
478          0.06994510442018509,
479          0.09231198579072952,
480          0.0953044667840004,
481          0.09653972834348679,
482          0.06668726354837418
483        ]
484      },
485      {
486        "load_cv": 0.3588871657819647,
487        "raw_cv": 0.19309554993951816,
488        "pressure": [
489          0.0671059787273407,
490          0.11245114356279373,
491          0.08675403892993927,
492          0.09586981683969498,
493          0.10887758433818817,
494          0.06107066944241524,
495          0.11346158385276794,
496          0.07197955995798111
497        ]
498      },
499      {
500        "load_cv": 0.5058655738790097,
501        "raw_cv": 0.1606991684840679,
502        "pressure": [
503          0.08636239171028137,
504          0.09632236510515213,
505          0.11678074300289154,
506          0.08211132884025574,
507          0.08124763518571854,
508          0.08318377286195755,
509          0.10013129562139511,
510          0.07143087685108185
511        ]
512      },
513      {
514        "load_cv": 0.34161379933083946,
515        "raw_cv": 0.12798243760960235,
516        "pressure": [
517          0.08367805182933807,
518          0.09391549974679947,
519          0.08226265013217926,
520          0.0783558338880539,
521          0.09979941695928574,
522          0.10702546685934067,
523          0.09416698664426804,
524          0.07836645841598511
525        ]
526      }
527    ],
528    "best_cfg": {
529      "lr": 0.006,
530      "alpha": 2.0,
531      "aux": 0.0,
532      "beta": 0.9,
533      "wd": 0.0,
534      "epochs": 3
535    },
536    "sweep": [
537      {
538        "cfg": {
539          "lr": 0.001,
540          "alpha": 0.5,
541          "aux": 0.0,
542          "beta": 0.9,
543          "wd": 0.0,
544          "epochs": 3
545        },
546        "mean": 0.3829677514731884
547      },
548      {
549        "cfg": {
550          "lr": 0.001,
551          "alpha": 1.0,
552          "aux": 0.0,
553          "beta": 0.9,
554          "wd": 0.0,
555          "epochs": 3
556        },
557        "mean": 0.38300157710909843
558      },
559      {
560        "cfg": {
561          "lr": 0.001,
562          "alpha": 2.0,
563          "aux": 0.0,
564          "beta": 0.9,
565          "wd": 0.0,
566          "epochs": 3
567        },
568        "mean": 0.3818103075027466
569      },
570      {
571        "cfg": {
572          "lr": 0.003,
573          "alpha": 0.5,
574          "aux": 0.0,
575          "beta": 0.9,
576          "wd": 0.0,
577          "epochs": 3
578        },
579        "mean": 0.3212699890136719
580      },
581      {
582        "cfg": {
583          "lr": 0.003,
584          "alpha": 1.0,
585          "aux": 0.0,
586          "beta": 0.9,
587          "wd": 0.0,
588          "epochs": 3
589        },
590        "mean": 0.32134268060326576
591      },
592      {
593        "cfg": {
594          "lr": 0.003,
595          "alpha": 2.0,
596          "aux": 0.0,
597          "beta": 0.9,
598          "wd": 0.0,
599          "epochs": 3
600        },
601        "mean": 0.32290053740143776
602      },
603      {
604        "cfg": {
605          "lr": 0.006,
606          "alpha": 0.5,
607          "aux": 0.0,
608          "beta": 0.9,
609          "wd": 0.0,
610          "epochs": 3
611        },
612        "mean": 0.3090713191777468
613      },
614      {
615        "cfg": {
616          "lr": 0.006,
617          "alpha": 1.0,
618          "aux": 0.0,
619          "beta": 0.9,
620          "wd": 0.0,
621          "epochs": 3
622        },
623        "mean": 0.3074618801474571
624      },
625      {
626        "cfg": {
627          "lr": 0.006,
628          "alpha": 2.0,
629          "aux": 0.0,
630          "beta": 0.9,
631          "wd": 0.0,
632          "epochs": 3
633        },
634        "mean": 0.30424891598522663
635      }
636    ]
637  },
638  "comparison": {
639    "delta_mean": 0.0012042541056871414,
640    "idea_wins": 3,
641    "n_pairs": 8,
642    "per_seed_diffs": [
643      0.006574094295501709,
644      0.0019710958003997803,
645      -0.03793981671333313,
646      0.026686102151870728,
647      0.01068153977394104,
648      -0.0048316121101379395,
649      -0.016457363963127136,
650      0.02294999361038208
651    ],
652    "p_value": 0.8799,
653    "mde": 0.01765867919995847,
654    "mde_rel_pct": 5.827088024067492,
655    "verdict": "no measurable effect",
656    "system_worked": false,
657    "paired_delta_mean": 0.0012042541056871414,
658    "permutation_pvalue": 0.8799
659  },
660  "mechanism_signature": {
661    "predicted_raw_pressure_cv_reduction": 0.17827335119625737,
662    "observed_trained_routed_load_cv": 0.42498467117208,
663    "confirmed": false
664  },
665  "custom_track": {
666    "name": "heterogeneous_regime_moe",
667    "file": "stage2_bench.py",
668    "domain": "moe-routing"
669  },
670  "runtime_sec": 99.08687734603882
671}