Expansion-balanced MoE routing / results.json
Mechanism confirmed, baseline not beaten
1{
2 "seed": 1430,
3 "verification": {
4 "formula_sweeps": [
5 {
6 "prediction": "uniform soft count",
7 "m": 4,
8 "observed": 3.310546875,
9 "predicted": 3.310546875,
10 "abs_error": 0.0
11 },
12 {
13 "prediction": "uniform soft count",
14 "m": 8,
15 "observed": 5.251128673553467,
16 "predicted": 5.251128673553467,
17 "abs_error": 0.0
18 },
19 {
20 "prediction": "uniform soft count",
21 "m": 16,
22 "observed": 7.055462837219238,
23 "predicted": 7.05546330383001,
24 "abs_error": 4.6661077135468076e-07
25 },
26 {
27 "prediction": "uniform soft count",
28 "m": 32,
29 "observed": 7.888481140136719,
30 "predicted": 7.888481303698535,
31 "abs_error": 1.6356181653520707e-07
32 },
33 {
34 "prediction": "q-expert finite-group soft count",
35 "q": 1,
36 "observed": 1.0,
37 "predicted": 1.0,
38 "abs_error": 0.0
39 },
40 {
41 "prediction": "q-expert finite-group soft count",
42 "q": 2,
43 "observed": 1.999969482421875,
44 "predicted": 1.999969482421875,
45 "abs_error": 0.0
46 },
47 {
48 "prediction": "q-expert finite-group soft count",
49 "q": 4,
50 "observed": 3.959909677505493,
51 "predicted": 3.959909616969526,
52 "abs_error": 6.05359673500061e-08
53 },
54 {
55 "prediction": "q-expert finite-group soft count",
56 "q": 8,
57 "observed": 7.055462837219238,
58 "predicted": 7.05546330383001,
59 "abs_error": 4.6661077135468076e-07
60 }
61 ],
62 "lambda_sweep": [
63 {
64 "lambda": 0,
65 "observed": 0.0,
66 "predicted": 0.0
67 },
68 {
69 "lambda": 0.1,
70 "observed": 0.18027440309524537,
71 "predicted": 0.18027440309524537
72 },
73 {
74 "lambda": 0.5,
75 "observed": 0.9013720154762268,
76 "predicted": 0.9013720154762268
77 },
78 {
79 "lambda": 1,
80 "observed": 1.8027440309524536,
81 "predicted": 1.8027440309524536
82 },
83 {
84 "lambda": 2,
85 "observed": 3.6054880619049072,
86 "predicted": 3.6054880619049072
87 },
88 {
89 "lambda": 5,
90 "observed": 9.013720154762268,
91 "predicted": 9.013720154762268
92 }
93 ],
94 "epsilon_threshold_sweep": [
95 {
96 "epsilon": 1.0536396232851564,
97 "predicted_hinge": 0.0,
98 "observed_hinge": 0.0
99 },
100 {
101 "epsilon": 2.1072792465703127,
102 "predicted_hinge": 0.0,
103 "observed_hinge": 0.0
104 },
105 {
106 "epsilon": 3.160918869855469,
107 "predicted_hinge": 6.893588086553852,
108 "observed_hinge": 6.893588086553852
109 }
110 ],
111 "epsilon_star": 2.1072792465703127,
112 "base_penalty": 1.8027440309524536
113 },
114 "baseline": {
115 "cross_entropy": 0.0012828032486140728,
116 "max_hard_load": 32,
117 "min_hard_load": 0,
118 "load_std": 16.0,
119 "dropped_fraction": 0.5,
120 "mean_soft_expansion_penalty": 4.3349175453186035,
121 "violation_fraction": 1.0,
122 "mean_hard_neighborhood": 3.8958333333333335,
123 "loss_trace": [
124 2.077996015548706,
125 0.1533023864030838,
126 0.15128406882286072
127 ]
128 },
129 "expansion_balanced": {
130 "cross_entropy": 0.18213726580142975,
131 "max_hard_load": 32,
132 "min_hard_load": 0,
133 "load_std": 13.076696830622021,
134 "dropped_fraction": 0.390625,
135 "mean_soft_expansion_penalty": 0.7138320803642273,
136 "violation_fraction": 0.2916666666666667,
137 "mean_hard_neighborhood": 6.0,
138 "loss_trace": [
139 2.077996015548706,
140 0.8379582166671753,
141 0.8262269496917725
142 ]
143 },
144 "notes": "n=128, E=8, top-1 diagnostic, capacity=16/expert, identical Adam setup; expansion uses fixed random subset schedule."
145}