Centered Heavy-Tail Clipping Optimizer / bench_report.json
Failed on benchmark
1{
2 "bench_version": 1,
3 "track": "tabular",
4 "model": "mlp_tiny",
5 "metric_direction": "lower is better",
6 "n_seeds": 8,
7 "baseline": {
8 "best_cfg": {
9 "lr": 0.024,
10 "tau": 1.0
11 },
12 "sweep": [
13 {
14 "cfg": {
15 "lr": 0.0015,
16 "tau": 0.5
17 },
18 "mean": 149.69311714172363
19 },
20 {
21 "cfg": {
22 "lr": 0.0015,
23 "tau": 1.0
24 },
25 "mean": 149.69156074523926
26 },
27 {
28 "cfg": {
29 "lr": 0.0015,
30 "tau": 2.0
31 },
32 "mean": 149.69087982177734
33 },
34 {
35 "cfg": {
36 "lr": 0.003,
37 "tau": 0.5
38 },
39 "mean": 20.230180501937866
40 },
41 {
42 "cfg": {
43 "lr": 0.003,
44 "tau": 1.0
45 },
46 "mean": 20.230008840560913
47 },
48 {
49 "cfg": {
50 "lr": 0.003,
51 "tau": 2.0
52 },
53 "mean": 20.22838020324707
54 },
55 {
56 "cfg": {
57 "lr": 0.006,
58 "tau": 0.5
59 },
60 "mean": 17.811020731925964
61 },
62 {
63 "cfg": {
64 "lr": 0.006,
65 "tau": 1.0
66 },
67 "mean": 17.814178228378296
68 },
69 {
70 "cfg": {
71 "lr": 0.006,
72 "tau": 2.0
73 },
74 "mean": 17.815396189689636
75 },
76 {
77 "cfg": {
78 "lr": 0.012,
79 "tau": 0.5
80 },
81 "mean": 13.617882013320923
82 },
83 {
84 "cfg": {
85 "lr": 0.012,
86 "tau": 1.0
87 },
88 "mean": 13.623129487037659
89 },
90 {
91 "cfg": {
92 "lr": 0.012,
93 "tau": 2.0
94 },
95 "mean": 13.62790322303772
96 },
97 {
98 "cfg": {
99 "lr": 0.024,
100 "tau": 0.5
101 },
102 "mean": 11.456546604633331
103 },
104 {
105 "cfg": {
106 "lr": 0.024,
107 "tau": 1.0
108 },
109 "mean": 11.436318755149841
110 },
111 {
112 "cfg": {
113 "lr": 0.024,
114 "tau": 2.0
115 },
116 "mean": 11.445708155632019
117 }
118 ],
119 "full": {
120 "mean": 11.436318755149841,
121 "std": 2.802007267234235,
122 "per_seed": [
123 13.539134979248047,
124 14.802740097045898,
125 14.924714088439941,
126 9.829378128051758,
127 7.340484619140625,
128 7.899506568908691,
129 10.356738090515137,
130 12.797853469848633
131 ],
132 "n": 8
133 }
134 },
135 "idea": {
136 "mean": 15.096654891967773,
137 "std": 1.5912454314949351,
138 "per_seed": [
139 14.8751220703125,
140 15.759033203125,
141 13.488835334777832,
142 14.056472778320312,
143 13.784052848815918,
144 13.798114776611328,
145 16.769433975219727,
146 18.24217414855957
147 ],
148 "n": 8,
149 "cfg": {
150 "lr": 0.024,
151 "tau": 1.0
152 }
153 },
154 "comparison": {
155 "delta_mean": 3.660336136817932,
156 "idea_wins": 1,
157 "n_pairs": 8,
158 "per_seed_diffs": [
159 1.3359870910644531,
160 0.9562931060791016,
161 -1.4358787536621094,
162 4.227094650268555,
163 6.443568229675293,
164 5.898608207702637,
165 6.41269588470459,
166 5.4443206787109375
167 ],
168 "p_value": 0.03245,
169 "mde": 2.498914897900942,
170 "mde_rel_pct": 21.85069296687508,
171 "verdict": "idea worse (significant)",
172 "system_worked": false
173 },
174 "idea_grid": [
175 {
176 "mean": 15.096654891967773,
177 "std": 1.5912454314949351,
178 "per_seed": [
179 14.8751220703125,
180 15.759033203125,
181 13.488835334777832,
182 14.056472778320312,
183 13.784052848815918,
184 13.798114776611328,
185 16.769433975219727,
186 18.24217414855957
187 ],
188 "n": 8,
189 "cfg": {
190 "lr": 0.024,
191 "tau": 1.0
192 }
193 },
194 {
195 "mean": 22.211764812469482,
196 "std": 2.486166287168737,
197 "per_seed": [
198 22.993654251098633,
199 21.551036834716797,
200 27.089385986328125,
201 18.64665412902832,
202 22.97047233581543,
203 18.93794059753418,
204 22.85433006286621,
205 22.650644302368164
206 ],
207 "n": 8,
208 "cfg": {
209 "lr": 0.006,
210 "tau": 1.0
211 }
212 },
213 {
214 "mean": 15.096654891967773,
215 "std": 1.5912454314949351,
216 "per_seed": [
217 14.8751220703125,
218 15.759033203125,
219 13.488835334777832,
220 14.056472778320312,
221 13.784052848815918,
222 13.798114776611328,
223 16.769433975219727,
224 18.24217414855957
225 ],
226 "n": 8,
227 "cfg": {
228 "lr": 0.024,
229 "tau": 1.0
230 }
231 }
232 ],
233 "paired_diffs_baseline_minus_idea": [
234 -1.3359870910644531,
235 -0.9562931060791016,
236 1.4358787536621094,
237 -4.227094650268555,
238 -6.443568229675293,
239 -5.898608207702637,
240 -6.41269588470459,
241 -5.4443206787109375
242 ],
243 "permutation_pvalue": 0.03245,
244 "mechanism_signature": {
245 "prediction": "residual norm <= tau and common-direction cosine remains near 1",
246 "predicted_residual_bound": 1.0,
247 "observed_max_residual_mean": 1.0000003725290298,
248 "observed_direction_cosine_mean": 0.5656541213393211,
249 "confirmed": false,
250 "per_seed": [
251 {
252 "metric": 14.8751220703125,
253 "observed_max_residual": 1.000000238418579,
254 "predicted_tau": 1.0,
255 "direction_cosine": 0.2306671440601349
256 },
257 {
258 "metric": 15.759033203125,
259 "observed_max_residual": 1.0000005960464478,
260 "predicted_tau": 1.0,
261 "direction_cosine": 0.8679510951042175
262 },
263 {
264 "metric": 13.488835334777832,
265 "observed_max_residual": 1.0000005960464478,
266 "predicted_tau": 1.0,
267 "direction_cosine": 0.8376753926277161
268 },
269 {
270 "metric": 14.056472778320312,
271 "observed_max_residual": 1.0000003576278687,
272 "predicted_tau": 1.0,
273 "direction_cosine": 0.16625230014324188
274 },
275 {
276 "metric": 13.784052848815918,
277 "observed_max_residual": 1.0000005960464478,
278 "predicted_tau": 1.0,
279 "direction_cosine": 0.8546982407569885
280 },
281 {
282 "metric": 13.798114776611328,
283 "observed_max_residual": 1.0000001192092896,
284 "predicted_tau": 1.0,
285 "direction_cosine": 0.8671970367431641
286 },
287 {
288 "metric": 16.769433975219727,
289 "observed_max_residual": 1.000000238418579,
290 "predicted_tau": 1.0,
291 "direction_cosine": 0.6522518992424011
292 },
293 {
294 "metric": 18.24217414855957,
295 "observed_max_residual": 1.000000238418579,
296 "predicted_tau": 1.0,
297 "direction_cosine": 0.04853986203670502
298 }
299 ]
300 }
301}