1{
  2  "bench_version": 1,
  3  "track": "correlated_token_moe_regression",
  4  "model": "custom_moe_shared",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "best_cfg": {
  9      "lr": 0.006,
 10      "eta": 0.03
 11    },
 12    "sweep": [
 13      {
 14        "cfg": {
 15          "lr": 0.001,
 16          "eta": 0.03
 17        },
 18        "mean": 0.06951743829995394
 19      },
 20      {
 21        "cfg": {
 22          "lr": 0.001,
 23          "eta": 0.1
 24        },
 25        "mean": 0.06951743829995394
 26      },
 27      {
 28        "cfg": {
 29          "lr": 0.001,
 30          "eta": 0.3
 31        },
 32        "mean": 0.06951743829995394
 33      },
 34      {
 35        "cfg": {
 36          "lr": 0.003,
 37          "eta": 0.03
 38        },
 39        "mean": 0.043228451162576675
 40      },
 41      {
 42        "cfg": {
 43          "lr": 0.003,
 44          "eta": 0.1
 45        },
 46        "mean": 0.043228451162576675
 47      },
 48      {
 49        "cfg": {
 50          "lr": 0.003,
 51          "eta": 0.3
 52        },
 53        "mean": 0.043228451162576675
 54      },
 55      {
 56        "cfg": {
 57          "lr": 0.006,
 58          "eta": 0.03
 59        },
 60        "mean": 0.04188114870339632
 61      },
 62      {
 63        "cfg": {
 64          "lr": 0.006,
 65          "eta": 0.1
 66        },
 67        "mean": 0.04188114870339632
 68      },
 69      {
 70        "cfg": {
 71          "lr": 0.006,
 72          "eta": 0.3
 73        },
 74        "mean": 0.04188114870339632
 75      }
 76    ],
 77    "full": {
 78      "mean": 0.04093059757724404,
 79      "std": 0.0038662018493219787,
 80      "per_seed": [
 81        0.036128461360931396,
 82        0.04280463978648186,
 83        0.04104797914624214,
 84        0.047543514519929886,
 85        0.038935478776693344,
 86        0.04488188028335571,
 87        0.04060979187488556,
 88        0.03549303486943245
 89      ],
 90      "n": 8
 91    }
 92  },
 93  "idea": {
 94    "mean": 0.04056028323248029,
 95    "std": 0.0035352381007606546,
 96    "per_seed": [
 97      0.03617550805211067,
 98      0.042327430099248886,
 99      0.040069837123155594,
100      0.04594426974654198,
101      0.039504799991846085,
102      0.04505091905593872,
103      0.04003051668405533,
104      0.03537898510694504
105    ],
106    "n": 8
107  },
108  "comparison": {
109    "delta_mean": -0.0003703143447637558,
110    "idea_wins": 5,
111    "n_pairs": 8,
112    "per_seed_diffs": [
113      4.704669117927551e-05,
114      -0.0004772096872329712,
115      -0.0009781420230865479,
116      -0.001599244773387909,
117      0.0005693212151527405,
118      0.0001690387725830078,
119      -0.0005792751908302307,
120      -0.0001140497624874115
121    ],
122    "p_value": 0.20205,
123    "mde": 0.0005783116111008068,
124    "mde_rel_pct": 1.412907813059459,
125    "verdict": "no measurable effect",
126    "system_worked": false
127  },
128  "mechanism_signature": {
129    "mechanism_signature": {
130      "predicted_prior_vs_observed_load_mae": 0.12510760004321733,
131      "predicted_prior_mean": [
132        0.4308023750782013,
133        0.25607896596193314,
134        0.31311867758631706
135      ],
136      "observed_load_mean": [
137        0.3356913700699806,
138        0.3363957405090332,
139        0.327912874519825
140      ],
141      "confirmed": false,
142      "note": "Measured on trained MoE systems; lower MAE means the coupled prior tracked realized routing load."
143    },
144    "selection": {
145      "baseline_union_grid": [
146        {
147          "lr": 0.001,
148          "eta": 0.03
149        },
150        {
151          "lr": 0.001,
152          "eta": 0.1
153        },
154        {
155          "lr": 0.001,
156          "eta": 0.3
157        },
158        {
159          "lr": 0.003,
160          "eta": 0.03
161        },
162        {
163          "lr": 0.003,
164          "eta": 0.1
165        },
166        {
167          "lr": 0.003,
168          "eta": 0.3
169        },
170        {
171          "lr": 0.006,
172          "eta": 0.03
173        },
174        {
175          "lr": 0.006,
176          "eta": 0.1
177        },
178        {
179          "lr": 0.006,
180          "eta": 0.3
181        }
182      ],
183      "idea_grid": [
184        {
185          "lr": 0.001,
186          "eta": 0.03
187        },
188        {
189          "lr": 0.003,
190          "eta": 0.1
191        },
192        {
193          "lr": 0.006,
194          "eta": 0.3
195        }
196      ],
197      "idea_best": {
198        "lr": 0.006,
199        "eta": 0.3
200      }
201    },
202    "per_seed_behavior": [
203      {
204        "seed": 0,
205        "baseline": {
206          "metric": 0.036128461360931396,
207          "load_var": 0.0028337526600807905,
208          "load": [
209            0.4048278331756592,
210            0.2771647572517395,
211            0.31800737977027893
212          ],
213          "train_load": [
214            0.4017528295516968,
215            0.2775239944458008,
216            0.32072314620018005
217          ],
218          "prior": [
219            0.29100728034973145,
220            0.3502899706363678,
221            0.35870274901390076
222          ]
223        },
224        "idea": {
225          "metric": 0.03617550805211067,
226          "load_var": 3.5828783438773826e-05,
227          "load": [
228            0.33977699279785156,
229            0.3253572881221771,
230            0.3348657488822937
231          ],
232          "train_load": [
233            0.3369426131248474,
234            0.32563209533691406,
235            0.33742526173591614
236          ],
237          "prior": [
238            0.2513711452484131,
239            0.3956768214702606,
240            0.35295209288597107
241          ]
242        }
243      },
244      {
245        "seed": 1,
246        "baseline": {
247          "metric": 0.04280463978648186,
248          "load_var": 0.001193343661725521,
249          "load": [
250            0.3492494523525238,
251            0.3653755486011505,
252            0.2853749692440033
253          ],
254          "train_load": [
255            0.35198989510536194,
256            0.3660563826560974,
257            0.2819536626338959
258          ],
259          "prior": [
260            0.3014691472053528,
261            0.3053934872150421,
262            0.3931373059749603
263          ]
264        },
265        "idea": {
266          "metric": 0.042327430099248886,
267          "load_var": 3.9590442611370236e-05,
268          "load": [
269            0.3245660364627838,
270            0.3390347957611084,
271            0.3363991379737854
272          ],
273          "train_load": [
274            0.3271709382534027,
275            0.339790940284729,
276            0.3330380916595459
277          ],
278          "prior": [
279            0.3157062232494354,
280            0.28578829765319824,
281            0.39850544929504395
282          ]
283        }
284      },
285      {
286        "seed": 2,
287        "baseline": {
288          "metric": 0.04104797914624214,
289          "load_var": 0.006669501308351755,
290          "load": [
291            0.2326611876487732,
292            0.43269073963165283,
293            0.3346480429172516
294          ],
295          "train_load": [
296            0.23457762598991394,
297            0.42982161045074463,
298            0.33560073375701904
299          ],
300          "prior": [
301            0.42684873938560486,
302            0.26154571771621704,
303            0.3116055727005005
304          ]
305        },
306        "idea": {
307          "metric": 0.040069837123155594,
308          "load_var": 7.834684765839484e-06,
309          "load": [
310            0.3340980112552643,
311            0.3295874297618866,
312            0.33631452918052673
313          ],
314          "train_load": [
315            0.33461418747901917,
316            0.32943210005760193,
317            0.3359537124633789
318          ],
319          "prior": [
320            0.5657042860984802,
321            0.17496956884860992,
322            0.25932615995407104
323          ]
324        }
325      },
326      {
327        "seed": 3,
328        "baseline": {
329          "metric": 0.047543514519929886,
330          "load_var": 0.006999241188168526,
331          "load": [
332            0.25769320130348206,
333            0.29236385226249695,
334            0.4499429166316986
335          ],
336          "train_load": [
337            0.25029802322387695,
338            0.30129286646842957,
339            0.4484090209007263
340          ],
341          "prior": [
342            0.37379252910614014,
343            0.3889646530151367,
344            0.23724278807640076
345          ]
346        },
347        "idea": {
348          "metric": 0.04594426974654198,
349          "load_var": 4.3429772631498054e-05,
350          "load": [
351            0.34203284978866577,
352            0.3318789601325989,
353            0.32608816027641296
354          ],
355          "train_load": [
356            0.3329542577266693,
357            0.344044029712677,
358            0.32300177216529846
359          ],
360          "prior": [
361            0.4113822877407074,
362            0.3852009177207947,
363            0.20341677963733673
364          ]
365        }
366      },
367      {
368        "seed": 4,
369        "baseline": {
370          "metric": 0.038935478776693344,
371          "load_var": 0.016052283346652985,
372          "load": [
373            0.492380827665329,
374            0.32526832818984985,
375            0.18235087394714355
376          ],
377          "train_load": [
378            0.49742141366004944,
379            0.3184661865234375,
380            0.18411238491535187
381          ],
382          "prior": [
383            0.23821482062339783,
384            0.34330517053604126,
385            0.4184800386428833
386          ]
387        },
388        "idea": {
389          "metric": 0.039504799991846085,
390          "load_var": 0.00030185282230377197,
391          "load": [
392            0.33665525913238525,
393            0.352755606174469,
394            0.31058913469314575
395          ],
396          "train_load": [
397            0.34037309885025024,
398            0.34567272663116455,
399            0.3139541447162628
400          ],
401          "prior": [
402            0.1704246699810028,
403            0.30951765179634094,
404            0.520057737827301
405          ]
406        }
407      },
408      {
409        "seed": 5,
410        "baseline": {
411          "metric": 0.04488188028335571,
412          "load_var": 0.012242220342159271,
413          "load": [
414            0.17938412725925446,
415            0.4345579147338867,
416            0.38605797290802
417          ],
418          "train_load": [
419            0.18139150738716125,
420            0.44180044531822205,
421            0.3768080174922943
422          ],
423          "prior": [
424            0.4594753384590149,
425            0.2737727165222168,
426            0.2667520344257355
427          ]
428        },
429        "idea": {
430          "metric": 0.04505091905593872,
431          "load_var": 3.8094429328339174e-05,
432          "load": [
433            0.3336072862148285,
434            0.3407518267631531,
435            0.32564085721969604
436          ],
437          "train_load": [
438            0.3371290862560272,
439            0.3452141582965851,
440            0.3176567256450653
441          ],
442          "prior": [
443            0.6375130414962769,
444            0.15396623313426971,
445            0.20852075517177582
446          ]
447        }
448      },
449      {
450        "seed": 6,
451        "baseline": {
452          "metric": 0.04060979187488556,
453          "load_var": 0.0106210857629776,
454          "load": [
455            0.18771888315677643,
456            0.4115215837955475,
457            0.4007594883441925
458          ],
459          "train_load": [
460            0.18947289884090424,
461            0.41077253222465515,
462            0.39975452423095703
463          ],
464          "prior": [
465            0.43537044525146484,
466            0.26841896772384644,
467            0.29621052742004395
468          ]
469        },
470        "idea": {
471          "metric": 0.04003051668405533,
472          "load_var": 9.865413449006155e-05,
473          "load": [
474            0.3200598955154419,
475            0.33598971366882324,
476            0.34395039081573486
477          ],
478          "train_load": [
479            0.3222788870334625,
480            0.3344203233718872,
481            0.3433007597923279
482          ],
483          "prior": [
484            0.6139684319496155,
485            0.1969204843044281,
486            0.1891111135482788
487          ]
488        }
489      },
490      {
491        "seed": 7,
492        "baseline": {
493          "metric": 0.03549303486943245,
494          "load_var": 0.01686282269656658,
495          "load": [
496            0.2574109137058258,
497            0.5161084532737732,
498            0.22648058831691742
499          ],
500          "train_load": [
501            0.2536790072917938,
502            0.5173004865646362,
503            0.22902047634124756
504          ],
505          "prior": [
506            0.40807998180389404,
507            0.21958644688129425,
508            0.3723335862159729
509          ]
510        },
511        "idea": {
512          "metric": 0.03537898510694504,
513          "load_var": 0.00034477459848858416,
514          "load": [
515            0.3547346293926239,
516            0.3358103036880493,
517            0.3094550371170044
518          ],
519          "train_load": [
520            0.3507271409034729,
521            0.33421364426612854,
522            0.31505918502807617
523          ],
524          "prior": [
525            0.4803489148616791,
526            0.14659175276756287,
527            0.37305933237075806
528          ]
529        }
530      }
531    ]
532  }
533}