Congestion-aware equimarginal MoE router / bench_report.json
Mechanism confirmed, baseline not beaten
1{
2 "bench_version": 1,
3 "track": "congestion_moe_sequence",
4 "model": "tiny_moe",
5 "metric_direction": "lower is better",
6 "n_seeds": 8,
7 "baseline": {
8 "best_cfg": {
9 "lr": 0.006,
10 "temperature": 0.5
11 },
12 "sweep": [
13 {
14 "cfg": {
15 "lr": 0.0015,
16 "temperature": 0.5
17 },
18 "mean": 1.013328194618225
19 },
20 {
21 "cfg": {
22 "lr": 0.0015,
23 "temperature": 0.7
24 },
25 "mean": 1.0138949006795883
26 },
27 {
28 "cfg": {
29 "lr": 0.0015,
30 "temperature": 1.0
31 },
32 "mean": 1.0139495730400085
33 },
34 {
35 "cfg": {
36 "lr": 0.003,
37 "temperature": 0.5
38 },
39 "mean": 0.9528659582138062
40 },
41 {
42 "cfg": {
43 "lr": 0.003,
44 "temperature": 0.7
45 },
46 "mean": 0.9548223316669464
47 },
48 {
49 "cfg": {
50 "lr": 0.003,
51 "temperature": 1.0
52 },
53 "mean": 0.9556002467870712
54 },
55 {
56 "cfg": {
57 "lr": 0.006,
58 "temperature": 0.5
59 },
60 "mean": 0.853488102555275
61 },
62 {
63 "cfg": {
64 "lr": 0.006,
65 "temperature": 0.7
66 },
67 "mean": 0.859209343791008
68 },
69 {
70 "cfg": {
71 "lr": 0.006,
72 "temperature": 1.0
73 },
74 "mean": 0.8618226200342178
75 }
76 ],
77 "full": {
78 "mean": 0.9469610154628754,
79 "std": 0.22496182254054511,
80 "per_seed": [
81 0.7249739170074463,
82 0.5741610527038574,
83 0.8283010125160217,
84 1.2865164279937744,
85 1.0606132745742798,
86 1.210940957069397,
87 0.897042989730835,
88 0.9931384921073914
89 ],
90 "n": 8
91 }
92 },
93 "idea": {
94 "mean": 0.9704978540539742,
95 "std": 0.21903613092802948,
96 "per_seed": [
97 0.7384425401687622,
98 0.5904383659362793,
99 0.8622050285339355,
100 1.2669295072555542,
101 1.1580994129180908,
102 1.1912760734558105,
103 0.947786271572113,
104 1.0088056325912476
105 ],
106 "n": 8
107 },
108 "comparison": {
109 "delta_mean": 0.023536838591098785,
110 "idea_wins": 2,
111 "n_pairs": 8,
112 "per_seed_diffs": [
113 0.013468623161315918,
114 0.016277313232421875,
115 0.03390401601791382,
116 -0.019586920738220215,
117 0.09748613834381104,
118 -0.019664883613586426,
119 0.050743281841278076,
120 0.0156671404838562
121 ],
122 "p_value": 0.13795,
123 "mde": 0.03200907134547522,
124 "mde_rel_pct": 3.380188922542831,
125 "verdict": "no significant win",
126 "system_worked": false
127 },
128 "mechanism_signature": {
129 "idea_sweep": [
130 {
131 "cfg": {
132 "lr": 0.0015,
133 "temperature": 0.7
134 },
135 "result": {
136 "mean": 1.1258931756019592,
137 "std": 0.2915624175884252,
138 "per_seed": [
139 0.8899675607681274,
140 0.7289124131202698,
141 0.9036810994148254,
142 1.533188819885254,
143 1.2896629571914673,
144 1.5818136930465698,
145 1.0076074600219727,
146 1.0723114013671875
147 ],
148 "n": 8
149 }
150 },
151 {
152 "cfg": {
153 "lr": 0.003,
154 "temperature": 0.7
155 },
156 "result": {
157 "mean": 1.0657112374901772,
158 "std": 0.26514025298024,
159 "per_seed": [
160 0.8271378874778748,
161 0.6704053282737732,
162 0.886384129524231,
163 1.442152738571167,
164 1.2345929145812988,
165 1.4378575086593628,
166 0.9755327105522156,
167 1.0516266822814941
168 ],
169 "n": 8
170 }
171 },
172 {
173 "cfg": {
174 "lr": 0.006,
175 "temperature": 0.7
176 },
177 "result": {
178 "mean": 0.9704978540539742,
179 "std": 0.21903613092802948,
180 "per_seed": [
181 0.7384425401687622,
182 0.5904383659362793,
183 0.8622050285339355,
184 1.2669295072555542,
185 1.1580994129180908,
186 1.1912760734558105,
187 0.947786271572113,
188 1.0088056325912476
189 ],
190 "n": 8
191 }
192 }
193 ],
194 "prediction": "congestion-aware routing reduces expert-load CV while retaining a fixed row budget",
195 "predicted_direction": "negative",
196 "observed_load_cv_delta": -0.28653995075335703,
197 "per_seed": [
198 {
199 "seed": 0,
200 "baseline_load_cv": 0.2926468500132308,
201 "idea_load_cv": 0.0714858382016512,
202 "baseline_mse": 0.7249739170074463,
203 "idea_mse": 0.7384425401687622
204 },
205 {
206 "seed": 1,
207 "baseline_load_cv": 0.521491851785781,
208 "idea_load_cv": 0.09097290992372925,
209 "baseline_mse": 0.5741610527038574,
210 "idea_mse": 0.5904383659362793
211 },
212 {
213 "seed": 2,
214 "baseline_load_cv": 0.4599689101988864,
215 "idea_load_cv": 0.08442225455900104,
216 "baseline_mse": 0.8283010125160217,
217 "idea_mse": 0.8622050285339355
218 },
219 {
220 "seed": 3,
221 "baseline_load_cv": 0.25446022032673565,
222 "idea_load_cv": 0.05582119941488141,
223 "baseline_mse": 1.2865164279937744,
224 "idea_mse": 1.2669295072555542
225 },
226 {
227 "seed": 4,
228 "baseline_load_cv": 0.49192661345027733,
229 "idea_load_cv": 0.08442110705389756,
230 "baseline_mse": 1.0606132745742798,
231 "idea_mse": 1.1580994129180908
232 },
233 {
234 "seed": 5,
235 "baseline_load_cv": 0.21399866103269982,
236 "idea_load_cv": 0.0416189082410498,
237 "baseline_mse": 1.210940957069397,
238 "idea_mse": 1.1912760734558105
239 },
240 {
241 "seed": 6,
242 "baseline_load_cv": 0.16870912551205047,
243 "idea_load_cv": 0.04076074600056684,
244 "baseline_mse": 0.897042989730835,
245 "idea_mse": 0.947786271572113
246 },
247 {
248 "seed": 7,
249 "baseline_load_cv": 0.43846935270462917,
250 "idea_load_cv": 0.07984901560265745,
251 "baseline_mse": 0.9931384921073914,
252 "idea_mse": 1.0088056325912476
253 }
254 ],
255 "confirmed": true
256 },
257 "custom_track": {
258 "name": "congestion_moe_sequence",
259 "file": "custom_moe_track.py",
260 "domain": "moe-routing"
261 },
262 "device": "cpu",
263 "epochs": 4,
264 "batch": 128
265}