1{
  2  "bench_version": 1,
  3  "track": "router_regime_regression",
  4  "model": "custom_mlp_moe",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "best_cfg": {
  9      "lr": 0.012,
 10      "temp": 0.7
 11    },
 12    "sweep": [
 13      {
 14        "cfg": {
 15          "lr": 0.003,
 16          "temp": 0.7
 17        },
 18        "mean": 0.5712137892842293
 19      },
 20      {
 21        "cfg": {
 22          "lr": 0.003,
 23          "temp": 1.0
 24        },
 25        "mean": 0.6234312355518341
 26      },
 27      {
 28        "cfg": {
 29          "lr": 0.006,
 30          "temp": 0.7
 31        },
 32        "mean": 0.4075416550040245
 33      },
 34      {
 35        "cfg": {
 36          "lr": 0.006,
 37          "temp": 1.0
 38        },
 39        "mean": 0.44819550961256027
 40      },
 41      {
 42        "cfg": {
 43          "lr": 0.012,
 44          "temp": 0.7
 45        },
 46        "mean": 0.27756691724061966
 47      },
 48      {
 49        "cfg": {
 50          "lr": 0.012,
 51          "temp": 1.0
 52        },
 53        "mean": 0.31893689185380936
 54      }
 55    ],
 56    "full": {
 57      "mean": 0.3102635517716408,
 58      "std": 0.07497470190378723,
 59      "per_seed": [
 60        0.30197641253471375,
 61        0.24877610802650452,
 62        0.29255229234695435,
 63        0.26696285605430603,
 64        0.28872963786125183,
 65        0.5001022219657898,
 66        0.26213279366493225,
 67        0.3208760917186737
 68      ],
 69      "n": 8
 70    }
 71  },
 72  "idea": {
 73    "cfg": {
 74      "lr": 0.012,
 75      "temp": 0.7,
 76      "rho": 0.3
 77    },
 78    "per_seed": [
 79      0.3346004784107208,
 80      0.27418217062950134,
 81      0.3511717915534973,
 82      0.32937872409820557,
 83      0.3155486285686493,
 84      0.5077916979789734,
 85      0.2751004695892334,
 86      0.34620028734207153
 87    ],
 88    "mean": 0.3417467810213566,
 89    "std": 0.07326728763277315,
 90    "signature_summary": {
 91      "mean_observed_cap": 0.2041564326380133,
 92      "mean_predicted_cap": 0.2041564326380133
 93    }
 94  },
 95  "comparison": {
 96    "delta_mean": 0.031483229249715805,
 97    "idea_wins": 0,
 98    "n_pairs": 8,
 99    "per_seed_diffs": [
100      0.03262406587600708,
101      0.025406062602996826,
102      0.05861949920654297,
103      0.062415868043899536,
104      0.02681899070739746,
105      0.007689476013183594,
106      0.012967675924301147,
107      0.025324195623397827
108    ],
109    "p_value": 0.0081,
110    "mde": 0.01642202588917077,
111    "mde_rel_pct": 5.292927833578616,
112    "verdict": "idea worse (significant)",
113    "system_worked": false
114  },
115  "mechanism_signature": {
116    "prediction": "resistance-scaled natural-gradient router updates obey the pairwise cap on trained NN updates",
117    "predicted_vs_observed": {
118      "mean_observed_cap": 0.2041564326380133,
119      "mean_predicted_cap": 0.2041564326380133
120    },
121    "confirmed": true,
122    "math_sanity": {
123      "max_resistance_ratio": 0.9998409991250671,
124      "min_covariance_bound_eigenvalue": -1.1862584540283742e-16,
125      "trust_cap_max_abs_error": 2.7755575615628914e-17,
126      "passed": true
127    }
128  },
129  "custom_track": {
130    "name": "router_regime_regression",
131    "file": "custom_router_track.py",
132    "domain": "moe-routing"
133  },
134  "idea_sweep": [
135    {
136      "cfg": {
137        "lr": 0.012,
138        "temp": 0.7,
139        "rho": 0.15
140      },
141      "per_seed": [
142        0.33006882667541504,
143        0.27459344267845154,
144        0.3564022481441498,
145        0.33827874064445496,
146        0.31755706667900085,
147        0.5151143074035645,
148        0.2753690779209137,
149        0.3360668420791626
150      ],
151      "mean": 0.3429313190281391,
152      "std": 0.07553734469571749,
153      "signature_summary": {
154        "mean_observed_cap": 0.13508778517103004,
155        "mean_predicted_cap": 0.13508778517103004
156      }
157    },
158    {
159      "cfg": {
160        "lr": 0.012,
161        "temp": 0.7,
162        "rho": 0.3
163      },
164      "per_seed": [
165        0.3346004784107208,
166        0.27418217062950134,
167        0.3511717915534973,
168        0.32937872409820557,
169        0.3155486285686493,
170        0.5077916979789734,
171        0.2751004695892334,
172        0.34620028734207153
173      ],
174      "mean": 0.3417467810213566,
175      "std": 0.07326728763277315,
176      "signature_summary": {
177        "mean_observed_cap": 0.2041564326380133,
178        "mean_predicted_cap": 0.2041564326380133
179      }
180    },
181    {
182      "cfg": {
183        "lr": 0.012,
184        "temp": 0.7,
185        "rho": 0.6
186      },
187      "per_seed": [
188        0.34508222341537476,
189        0.2762308120727539,
190        0.3442482650279999,
191        0.323551744222641,
192        0.31441816687583923,
193        0.5117174983024597,
194        0.2795194685459137,
195        0.36291345953941345
196      ],
197      "mean": 0.34471020475029945,
198      "std": 0.07418865304996113,
199      "signature_summary": {
200        "mean_observed_cap": 0.23960371281639814,
201        "mean_predicted_cap": 0.23960371281639814
202      }
203    },
204    {
205      "cfg": {
206        "lr": 0.003,
207        "temp": 0.7,
208        "rho": 0.3
209      },
210      "per_seed": [
211        0.6505195498466492,
212        0.5359265208244324,
213        0.6745683550834656,
214        0.5784613490104675,
215        0.6141240000724792,
216        0.7810488343238831,
217        0.5857309103012085,
218        0.6731484532356262
219      ],
220      "mean": 0.6366909965872765,
221      "std": 0.07601186541945128,
222      "signature_summary": {
223        "mean_observed_cap": 0.17640141463740155,
224        "mean_predicted_cap": 0.17640141463740155
225      }
226    },
227    {
228      "cfg": {
229        "lr": 0.006,
230        "temp": 0.7,
231        "rho": 0.3
232      },
233      "per_seed": [
234        0.49773088097572327,
235        0.380596786737442,
236        0.49826493859291077,
237        0.43299683928489685,
238        0.4217953383922577,
239        0.6373190879821777,
240        0.4525613486766815,
241        0.5051671862602234
242      ],
243      "mean": 0.47830405086278915,
244      "std": 0.07767939883669912,
245      "signature_summary": {
246        "mean_observed_cap": 0.19019251237068713,
247        "mean_predicted_cap": 0.19019251237068713
248      }
249    }
250  ]
251}