{
  "bench_version": 1,
  "track": "correlated_token_moe_regression",
  "model": "custom_moe",
  "metric_direction": "lower is better",
  "n_seeds": 8,
  "baseline": {
    "best_cfg": {
      "lr": 0.006,
      "aux": 0.01,
      "balanced": false,
      "K": 4
    },
    "sweep": [
      {
        "cfg": {
          "lr": 0.0015,
          "aux": 0.0,
          "balanced": false,
          "K": 4
        },
        "mean": 0.0866615679115057,
        "per_seed": [
          0.08718974888324738,
          0.09299269318580627,
          0.08348067104816437,
          0.08298315852880478
        ]
      },
      {
        "cfg": {
          "lr": 0.0015,
          "aux": 0.01,
          "balanced": false,
          "K": 4
        },
        "mean": 0.08026662468910217,
        "per_seed": [
          0.06962176412343979,
          0.09358783811330795,
          0.08609369397163391,
          0.07176320254802704
        ]
      },
      {
        "cfg": {
          "lr": 0.0015,
          "aux": 0.1,
          "balanced": false,
          "K": 4
        },
        "mean": 0.0802982971072197,
        "per_seed": [
          0.06983038038015366,
          0.09358862042427063,
          0.08610628545284271,
          0.07166790217161179
        ]
      },
      {
        "cfg": {
          "lr": 0.003,
          "aux": 0.0,
          "balanced": false,
          "K": 4
        },
        "mean": 0.07278427481651306,
        "per_seed": [
          0.07727840542793274,
          0.07310336083173752,
          0.07483050227165222,
          0.06592483073472977
        ]
      },
      {
        "cfg": {
          "lr": 0.003,
          "aux": 0.01,
          "balanced": false,
          "K": 4
        },
        "mean": 0.06707096565514803,
        "per_seed": [
          0.06202782317996025,
          0.07177857309579849,
          0.07393620163202286,
          0.060541264712810516
        ]
      },
      {
        "cfg": {
          "lr": 0.003,
          "aux": 0.1,
          "balanced": false,
          "K": 4
        },
        "mean": 0.06711035128682852,
        "per_seed": [
          0.062259428203105927,
          0.07183852046728134,
          0.0737687274813652,
          0.0605747289955616
        ]
      },
      {
        "cfg": {
          "lr": 0.006,
          "aux": 0.0,
          "balanced": false,
          "K": 4
        },
        "mean": 0.05864693410694599,
        "per_seed": [
          0.062149204313755035,
          0.05799993872642517,
          0.06060343608260155,
          0.05383515730500221
        ]
      },
      {
        "cfg": {
          "lr": 0.006,
          "aux": 0.01,
          "balanced": false,
          "K": 4
        },
        "mean": 0.056222427636384964,
        "per_seed": [
          0.049452465027570724,
          0.058252543210983276,
          0.05263408645987511,
          0.06455061584711075
        ]
      },
      {
        "cfg": {
          "lr": 0.006,
          "aux": 0.1,
          "balanced": false,
          "K": 4
        },
        "mean": 0.0565244285389781,
        "per_seed": [
          0.04950065538287163,
          0.0584375374019146,
          0.052471455186605453,
          0.06568806618452072
        ]
      }
    ],
    "full": {
      "per_seed": [
        0.049452465027570724,
        0.058252543210983276,
        0.05263408645987511,
        0.06455061584711075,
        0.053144052624702454,
        0.07924187928438187,
        0.04448141157627106,
        0.05137491226196289
      ],
      "mean": 0.056641495786607265,
      "records": [
        {
          "seed": 0,
          "metric": 0.049452465027570724,
          "routing": {
            "load_cv": 0.0913184674641444,
            "loads": [
              346.0,
              426.0,
              354.0,
              395.0,
              422.0,
              452.0,
              373.0,
              432.0
            ],
            "max_shell_count_spread": 311,
            "reassigned_fraction": 0.0,
            "shells": 4
          }
        },
        {
          "seed": 1,
          "metric": 0.058252543210983276,
          "routing": {
            "load_cv": 0.13504628836069504,
            "loads": [
              382.0,
              403.0,
              373.0,
              531.0,
              386.0,
              400.0,
              395.0,
              330.0
            ],
            "max_shell_count_spread": 184,
            "reassigned_fraction": 0.0,
            "shells": 4
          }
        },
        {
          "seed": 2,
          "metric": 0.05263408645987511,
          "routing": {
            "load_cv": 0.12644168616401794,
            "loads": [
              397.0,
              428.0,
              348.0,
              306.0,
              433.0,
              424.0,
              385.0,
              479.0
            ],
            "max_shell_count_spread": 159,
            "reassigned_fraction": 0.0,
            "shells": 4
          }
        },
        {
          "seed": 3,
          "metric": 0.06455061584711075,
          "routing": {
            "load_cv": 0.06754628043053149,
            "loads": [
              398.0,
              374.0,
              455.0,
              379.0,
              394.0,
              387.0,
              433.0,
              380.0
            ],
            "max_shell_count_spread": 269,
            "reassigned_fraction": 0.0,
            "shells": 4
          }
        },
        {
          "seed": 4,
          "metric": 0.053144052624702454,
          "routing": {
            "load_cv": 0.1465595015684756,
            "loads": [
              412.0,
              382.0,
              296.0,
              394.0,
              490.0,
              373.0,
              372.0,
              481.0
            ],
            "max_shell_count_spread": 297,
            "reassigned_fraction": 0.0,
            "shells": 4
          }
        },
        {
          "seed": 5,
          "metric": 0.07924187928438187,
          "routing": {
            "load_cv": 0.17972200755611428,
            "loads": [
              437.0,
              391.0,
              372.0,
              424.0,
              316.0,
              562.0,
              353.0,
              345.0
            ],
            "max_shell_count_spread": 321,
            "reassigned_fraction": 0.0,
            "shells": 4
          }
        },
        {
          "seed": 6,
          "metric": 0.04448141157627106,
          "routing": {
            "load_cv": 0.08075812652606547,
            "loads": [
              360.0,
              407.0,
              417.0,
              424.0,
              348.0,
              376.0,
              445.0,
              423.0
            ],
            "max_shell_count_spread": 264,
            "reassigned_fraction": 0.0,
            "shells": 4
          }
        },
        {
          "seed": 7,
          "metric": 0.05137491226196289,
          "routing": {
            "load_cv": 0.10753633107001559,
            "loads": [
              341.0,
              418.0,
              375.0,
              380.0,
              347.0,
              427.0,
              465.0,
              447.0
            ],
            "max_shell_count_spread": 262,
            "reassigned_fraction": 0.0,
            "shells": 4
          }
        }
      ],
      "cfg": {
        "lr": 0.006,
        "aux": 0.01,
        "balanced": false,
        "K": 4
      }
    }
  },
  "idea": {
    "per_seed": [
      0.05467335134744644,
      0.05477738380432129,
      0.056872498244047165,
      0.051517780870199203,
      0.05167350545525551,
      0.06755945831537247,
      0.04719924181699753,
      0.04966289922595024
    ],
    "mean": 0.05424201488494873,
    "records": [
      {
        "seed": 0,
        "metric": 0.05467335134744644,
        "routing": {
          "load_cv": 0.0,
          "loads": [
            400.0,
            400.0,
            400.0,
            400.0,
            400.0,
            400.0,
            400.0,
            400.0
          ],
          "max_shell_count_spread": 0,
          "reassigned_fraction": 0.61625,
          "shells": 8
        }
      },
      {
        "seed": 1,
        "metric": 0.05477738380432129,
        "routing": {
          "load_cv": 0.0,
          "loads": [
            400.0,
            400.0,
            400.0,
            400.0,
            400.0,
            400.0,
            400.0,
            400.0
          ],
          "max_shell_count_spread": 0,
          "reassigned_fraction": 0.3775,
          "shells": 8
        }
      },
      {
        "seed": 2,
        "metric": 0.056872498244047165,
        "routing": {
          "load_cv": 0.0,
          "loads": [
            400.0,
            400.0,
            400.0,
            400.0,
            400.0,
            400.0,
            400.0,
            400.0
          ],
          "max_shell_count_spread": 0,
          "reassigned_fraction": 0.5353125,
          "shells": 8
        }
      },
      {
        "seed": 3,
        "metric": 0.051517780870199203,
        "routing": {
          "load_cv": 0.0,
          "loads": [
            400.0,
            400.0,
            400.0,
            400.0,
            400.0,
            400.0,
            400.0,
            400.0
          ],
          "max_shell_count_spread": 0,
          "reassigned_fraction": 0.5540625,
          "shells": 8
        }
      },
      {
        "seed": 4,
        "metric": 0.05167350545525551,
        "routing": {
          "load_cv": 0.0,
          "loads": [
            400.0,
            400.0,
            400.0,
            400.0,
            400.0,
            400.0,
            400.0,
            400.0
          ],
          "max_shell_count_spread": 0,
          "reassigned_fraction": 0.4,
          "shells": 8
        }
      },
      {
        "seed": 5,
        "metric": 0.06755945831537247,
        "routing": {
          "load_cv": 0.0,
          "loads": [
            400.0,
            400.0,
            400.0,
            400.0,
            400.0,
            400.0,
            400.0,
            400.0
          ],
          "max_shell_count_spread": 0,
          "reassigned_fraction": 0.5334375,
          "shells": 8
        }
      },
      {
        "seed": 6,
        "metric": 0.04719924181699753,
        "routing": {
          "load_cv": 0.0,
          "loads": [
            400.0,
            400.0,
            400.0,
            400.0,
            400.0,
            400.0,
            400.0,
            400.0
          ],
          "max_shell_count_spread": 0,
          "reassigned_fraction": 0.534375,
          "shells": 8
        }
      },
      {
        "seed": 7,
        "metric": 0.04966289922595024,
        "routing": {
          "load_cv": 0.0,
          "loads": [
            400.0,
            400.0,
            400.0,
            400.0,
            400.0,
            400.0,
            400.0,
            400.0
          ],
          "max_shell_count_spread": 0,
          "reassigned_fraction": 0.4796875,
          "shells": 8
        }
      }
    ],
    "cfg": {
      "lr": 0.006,
      "balanced": true,
      "K": 8
    }
  },
  "comparison": {
    "delta_mean": -0.002399480901658535,
    "idea_wins": 5,
    "n_pairs": 8,
    "per_seed_diffs": [
      0.005220886319875717,
      -0.0034751594066619873,
      0.004238411784172058,
      -0.013032834976911545,
      -0.0014705471694469452,
      -0.0116824209690094,
      0.002717830240726471,
      -0.0017120130360126495
    ],
    "p_value": 0.38035,
    "mde": 0.00574154155483705,
    "mde_rel_pct": 10.136634767676144,
    "verdict": "no significant win",
    "system_worked": false
  },
  "mechanism_signature": {
    "custom_track": {
      "name": "correlated_token_moe_regression",
      "file": "shell_moe_bench_track.py",
      "domain": "moe-routing"
    },
    "prediction": "per-shell expert count spread <= 1",
    "baseline_mean_load_cv": 0.11686608614250746,
    "idea_mean_load_cv": 0.0,
    "baseline_mean_shell_spread": 258.375,
    "idea_mean_shell_spread": 0.0,
    "idea_mean_reassigned_fraction": 0.5038281250000001,
    "confirmed": true
  },
  "idea_trials": [
    {
      "cfg": {
        "lr": 0.0015,
        "balanced": true,
        "K": 2
      },
      "mean": 0.08199466113001108,
      "per_seed": [
        0.07786157727241516,
        0.0871136337518692,
        0.08436500281095505,
        0.07892747968435287,
        0.07485600560903549,
        0.10391078889369965,
        0.07056240737438202,
        0.07836039364337921
      ]
    },
    {
      "cfg": {
        "lr": 0.0015,
        "balanced": true,
        "K": 4
      },
      "mean": 0.08325883187353611,
      "per_seed": [
        0.07747699320316315,
        0.09112893790006638,
        0.08618856221437454,
        0.07941422611474991,
        0.07391377538442612,
        0.10588844120502472,
        0.07102843374013901,
        0.08103128522634506
      ]
    },
    {
      "cfg": {
        "lr": 0.0015,
        "balanced": true,
        "K": 8
      },
      "mean": 0.08230428863316774,
      "per_seed": [
        0.07840844988822937,
        0.08882392942905426,
        0.08409003913402557,
        0.08046981692314148,
        0.07284295558929443,
        0.10753589123487473,
        0.06986328959465027,
        0.07639993727207184
      ]
    },
    {
      "cfg": {
        "lr": 0.003,
        "balanced": true,
        "K": 2
      },
      "mean": 0.06749503640457988,
      "per_seed": [
        0.06871458142995834,
        0.06850986182689667,
        0.07267368584871292,
        0.06359779834747314,
        0.06259164959192276,
        0.08258448541164398,
        0.05686591938138008,
        0.06442230939865112
      ]
    },
    {
      "cfg": {
        "lr": 0.003,
        "balanced": true,
        "K": 4
      },
      "mean": 0.06823915103450418,
      "per_seed": [
        0.0692436695098877,
        0.07023227959871292,
        0.07335231453180313,
        0.061882417649030685,
        0.06339849531650543,
        0.08482187241315842,
        0.057540178298950195,
        0.06544198095798492
      ]
    },
    {
      "cfg": {
        "lr": 0.003,
        "balanced": true,
        "K": 8
      },
      "mean": 0.06749762082472444,
      "per_seed": [
        0.06905151903629303,
        0.06874477118253708,
        0.07229793071746826,
        0.06277107447385788,
        0.061685409396886826,
        0.08680634945631027,
        0.05603644251823425,
        0.06258746981620789
      ]
    },
    {
      "cfg": {
        "lr": 0.006,
        "balanced": true,
        "K": 2
      },
      "mean": 0.054870990104973316,
      "per_seed": [
        0.05411909893155098,
        0.057356927543878555,
        0.05673087760806084,
        0.055093344300985336,
        0.051891688257455826,
        0.06572796404361725,
        0.04767368361353874,
        0.050374336540699005
      ]
    },
    {
      "cfg": {
        "lr": 0.006,
        "balanced": true,
        "K": 4
      },
      "mean": 0.05486724339425564,
      "per_seed": [
        0.05602085962891579,
        0.056902941316366196,
        0.05770782381296158,
        0.052157238125801086,
        0.054022110998630524,
        0.06586991250514984,
        0.046959299594163895,
        0.0492977611720562
      ]
    },
    {
      "cfg": {
        "lr": 0.006,
        "balanced": true,
        "K": 8
      },
      "mean": 0.05424201488494873,
      "per_seed": [
        0.05467335134744644,
        0.05477738380432129,
        0.056872498244047165,
        0.051517780870199203,
        0.05167350545525551,
        0.06755945831537247,
        0.04719924181699753,
        0.04966289922595024
      ]
    }
  ]
}