1{
  2  "bench_version": 1,
  3  "track": "correlated_token_moe_regression",
  4  "model": "matched_custom",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "best_cfg": {
  9      "lr": 0.001,
 10      "epochs": 20
 11    },
 12    "sweep": [
 13      {
 14        "cfg": {
 15          "lr": 0.001,
 16          "epochs": 20
 17        },
 18        "mean": 0.05792584456503391
 19      },
 20      {
 21        "cfg": {
 22          "lr": 0.003,
 23          "epochs": 20
 24        },
 25        "mean": 0.0597206624224782
 26      },
 27      {
 28        "cfg": {
 29          "lr": 0.01,
 30          "epochs": 20
 31        },
 32        "mean": 0.06360050570219755
 33      }
 34    ],
 35    "full": {
 36      "mean": 0.05597660783678293,
 37      "std": 0.004513427280072455,
 38      "per_seed": [
 39        0.04968653619289398,
 40        0.060288313776254654,
 41        0.06273896247148514,
 42        0.05898956581950188,
 43        0.051729243248701096,
 44        0.05691293999552727,
 45        0.056976065039634705,
 46        0.05049123615026474
 47      ],
 48      "n": 8
 49    }
 50  },
 51  "idea": {
 52    "mean": 0.043164961505681276,
 53    "std": 0.004883880354277062,
 54    "per_seed": [
 55      0.03658263012766838,
 56      0.043850693851709366,
 57      0.04236328974366188,
 58      0.04947901517152786,
 59      0.041660089045763016,
 60      0.051025427877902985,
 61      0.04368860274553299,
 62      0.03666994348168373
 63    ],
 64    "n": 8
 65  },
 66  "comparison": {
 67    "delta_mean": -0.012811646331101656,
 68    "idea_wins": 8,
 69    "n_pairs": 8,
 70    "per_seed_diffs": [
 71      -0.013103906065225601,
 72      -0.016437619924545288,
 73      -0.020375672727823257,
 74      -0.009510550647974014,
 75      -0.01006915420293808,
 76      -0.005887512117624283,
 77      -0.013287462294101715,
 78      -0.013821292668581009
 79    ],
 80    "p_value": 0.0081,
 81    "mde": 0.0037133339468910474,
 82    "mde_rel_pct": 6.633724497415811,
 83    "verdict": "idea better (significant)",
 84    "system_worked": true
 85  },
 86  "mechanism_signature": {
 87    "quantity": "mean absolute prediction change under swapping two trained-task tokens",
 88    "prediction": "equivariant/invariant system has zero change while flattened baseline is nonzero",
 89    "baseline_per_seed": [
 90      0.02555830031633377,
 91      0.036283940076828,
 92      0.041776224970817566,
 93      0.06388069689273834,
 94      0.04568406194448471,
 95      0.04635582119226456,
 96      0.057599522173404694,
 97      0.04444808512926102
 98    ],
 99    "idea_per_seed": [
100      1.4726538211107254e-08,
101      1.4726538211107254e-08,
102      9.284121915698051e-09,
103      1.7113052308559418e-08,
104      1.3271346688270569e-08,
105      9.080395102500916e-09,
106      1.2281816452741623e-08,
107      1.0826624929904938e-08
108    ],
109    "baseline_mean": 0.04519833158701658,
110    "idea_mean": 1.2663804227486253e-08,
111    "confirmed": true
112  },
113  "idea_sweep": [
114    {
115      "cfg": {
116        "lr": 0.001,
117        "epochs": 20
118      },
119      "mean": 0.0434781601652503,
120      "std": 0.004274304660454058,
121      "per_seed": [
122        0.038152437657117844,
123        0.046353816986083984,
124        0.04167266935110092,
125        0.04806140437722206,
126        0.04175148531794548,
127        0.05084418132901192,
128        0.042754773050546646,
129        0.038234513252973557
130      ]
131    },
132    {
133      "cfg": {
134        "lr": 0.003,
135        "epochs": 20
136      },
137      "mean": 0.043164961505681276,
138      "std": 0.004883880354277062,
139      "per_seed": [
140        0.03658263012766838,
141        0.043850693851709366,
142        0.04236328974366188,
143        0.04947901517152786,
144        0.041660089045763016,
145        0.051025427877902985,
146        0.04368860274553299,
147        0.03666994348168373
148      ]
149    },
150    {
151      "cfg": {
152        "lr": 0.01,
153        "epochs": 20
154      },
155      "mean": 0.04354450246319175,
156      "std": 0.004289397371544806,
157      "per_seed": [
158        0.03576800599694252,
159        0.04776814952492714,
160        0.04307476058602333,
161        0.04668940603733063,
162        0.04327880218625069,
163        0.04976873844861984,
164        0.0428813174366951,
165        0.039126839488744736
166      ]
167    }
168  ],
169  "track_justification": "The registered correlated-token track contains exchangeable multi-token correlations; built-in tracks do not."
170}