Prolate Energy-Preserving Bottleneck / bench_report.json
Failed on benchmark
1{
2 "bench_version": 1,
3 "track": "sequence",
4 "model": "transformer_tiny",
5 "metric_direction": "lower is better",
6 "n_seeds": 8,
7 "baseline": {
8 "best_cfg": {
9 "lr": 0.001,
10 "rank": 32
11 },
12 "sweep": [
13 {
14 "cfg": {
15 "lr": 0.001,
16 "rank": 32
17 },
18 "mean": 0.3797149360179901
19 },
20 {
21 "cfg": {
22 "lr": 0.003,
23 "rank": 32
24 },
25 "mean": 0.5440608412027359
26 },
27 {
28 "cfg": {
29 "lr": 0.005,
30 "rank": 32
31 },
32 "mean": 0.6877395510673523
33 }
34 ],
35 "full": {
36 "mean": 0.3990439400076866,
37 "std": 0.03555091686816573,
38 "per_seed": [
39 0.40484797954559326,
40 0.35254013538360596,
41 0.3537353277206421,
42 0.40773630142211914,
43 0.3955448567867279,
44 0.39521750807762146,
45 0.4743589758872986,
46 0.4083704352378845
47 ],
48 "n": 8
49 }
50 },
51 "idea": {
52 "mean": 0.4859609939157963,
53 "std": 0.030690392503688,
54 "per_seed": [
55 0.5398046374320984,
56 0.4721372127532959,
57 0.4599805176258087,
58 0.4985331892967224,
59 0.46909099817276,
60 0.45167359709739685,
61 0.5286514163017273,
62 0.46781638264656067
63 ],
64 "n": 8
65 },
66 "comparison": {
67 "delta_mean": 0.08691705390810966,
68 "idea_wins": 0,
69 "n_pairs": 8,
70 "per_seed_diffs": [
71 0.13495665788650513,
72 0.11959707736968994,
73 0.10624518990516663,
74 0.09079688787460327,
75 0.0735461413860321,
76 0.05645608901977539,
77 0.05429244041442871,
78 0.05944594740867615
79 ],
80 "p_value": 0.0081,
81 "mde": 0.025849569428390114,
82 "mde_rel_pct": 6.477875450982211,
83 "verdict": "idea worse (significant)",
84 "system_worked": false
85 },
86 "idea_sweep": [
87 {
88 "cfg": {
89 "lr": 0.001,
90 "rank": 12
91 },
92 "result": {
93 "mean": 0.8132649660110474,
94 "std": 0.07039976968716767,
95 "per_seed": [
96 0.8669847846031189,
97 0.6813386082649231,
98 0.7922707200050354,
99 0.7799481153488159,
100 0.8783856630325317,
101 0.7520660758018494,
102 0.8496934175491333,
103 0.9054323434829712
104 ],
105 "n": 8
106 }
107 },
108 {
109 "cfg": {
110 "lr": 0.001,
111 "rank": 16
112 },
113 "result": {
114 "mean": 0.6447810083627701,
115 "std": 0.0902293737866928,
116 "per_seed": [
117 0.8473581075668335,
118 0.532031774520874,
119 0.5603737235069275,
120 0.6733526587486267,
121 0.6476457715034485,
122 0.6398608684539795,
123 0.6668751835823059,
124 0.590749979019165
125 ],
126 "n": 8
127 }
128 },
129 {
130 "cfg": {
131 "lr": 0.001,
132 "rank": 20
133 },
134 "result": {
135 "mean": 0.4859609939157963,
136 "std": 0.030690392503688,
137 "per_seed": [
138 0.5398046374320984,
139 0.4721372127532959,
140 0.4599805176258087,
141 0.4985331892967224,
142 0.46909099817276,
143 0.45167359709739685,
144 0.5286514163017273,
145 0.46781638264656067
146 ],
147 "n": 8
148 }
149 },
150 {
151 "cfg": {
152 "lr": 0.003,
153 "rank": 12
154 },
155 "result": {
156 "mean": 0.8662505745887756,
157 "std": 0.08758038922827249,
158 "per_seed": [
159 0.9283100366592407,
160 0.69541335105896,
161 0.9051690101623535,
162 0.8301766514778137,
163 0.9451277852058411,
164 0.7725451588630676,
165 0.9667305946350098,
166 0.8865320086479187
167 ],
168 "n": 8
169 }
170 },
171 {
172 "cfg": {
173 "lr": 0.003,
174 "rank": 16
175 },
176 "result": {
177 "mean": 0.6834343373775482,
178 "std": 0.09683352283532777,
179 "per_seed": [
180 0.8036410808563232,
181 0.5426891446113586,
182 0.5894339084625244,
183 0.8408225774765015,
184 0.6754513382911682,
185 0.6724597215652466,
186 0.7292736768722534,
187 0.6137032508850098
188 ],
189 "n": 8
190 }
191 },
192 {
193 "cfg": {
194 "lr": 0.003,
195 "rank": 20
196 },
197 "result": {
198 "mean": 0.5512603707611561,
199 "std": 0.09140216767338745,
200 "per_seed": [
201 0.7226142287254333,
202 0.4266541600227356,
203 0.5001299381256104,
204 0.5239855051040649,
205 0.459748238325119,
206 0.5338931083679199,
207 0.6257912516593933,
208 0.6172665357589722
209 ],
210 "n": 8
211 }
212 },
213 {
214 "cfg": {
215 "lr": 0.005,
216 "rank": 12
217 },
218 "result": {
219 "mean": 0.9177583232522011,
220 "std": 0.11157952187308297,
221 "per_seed": [
222 1.0815995931625366,
223 0.7145638465881348,
224 0.9597082734107971,
225 0.8518447875976562,
226 1.0344592332839966,
227 0.8185186982154846,
228 0.9612360000610352,
229 0.9201361536979675
230 ],
231 "n": 8
232 }
233 },
234 {
235 "cfg": {
236 "lr": 0.005,
237 "rank": 16
238 },
239 "result": {
240 "mean": 0.768573522567749,
241 "std": 0.1272932016139664,
242 "per_seed": [
243 0.9035587906837463,
244 0.5587367415428162,
245 0.6324987411499023,
246 0.8873728513717651,
247 0.7445271611213684,
248 0.9305623769760132,
249 0.804309606552124,
250 0.6870219111442566
251 ],
252 "n": 8
253 }
254 },
255 {
256 "cfg": {
257 "lr": 0.005,
258 "rank": 20
259 },
260 "result": {
261 "mean": 0.5918549560010433,
262 "std": 0.11067653453007016,
263 "per_seed": [
264 0.8396977186203003,
265 0.4893060028553009,
266 0.5174769163131714,
267 0.6687707304954529,
268 0.5009192824363708,
269 0.5292778015136719,
270 0.6293551921844482,
271 0.5600360035896301
272 ],
273 "n": 8
274 }
275 }
276 ],
277 "audit": {
278 "structural_match": "multi-token sequence forecast",
279 "theorem_rank_asymptotic": 15.311856320206266,
280 "empirical_rank": 15,
281 "chosen_rank_grid": [
282 12,
283 16,
284 20
285 ],
286 "epochs": 8,
287 "n_train": 800,
288 "n_test": 300
289 },
290 "mechanism_signature": {
291 "prediction": "DPSS reconstruction error < Fourier at theorem rank",
292 "predicted_dpss_lt_fourier": true,
293 "observed_dpss_error_mean": 0.21955212997938756,
294 "observed_fourier_error_mean": 0.20034139348637556,
295 "observed_relative_improvement": -0.0958900013557031,
296 "attention_work_fraction": 0.25,
297 "trained_model_samples": [
298 {
299 "seed": 0,
300 "dpss_reconstruction_error": 0.22673050321714594,
301 "fourier_reconstruction_error": 0.20555503007570636,
302 "prediction_rms": 0.6146740913391113
303 },
304 {
305 "seed": 1,
306 "dpss_reconstruction_error": 0.221470346993945,
307 "fourier_reconstruction_error": 0.1998581531341289,
308 "prediction_rms": 0.6230881810188293
309 },
310 {
311 "seed": 2,
312 "dpss_reconstruction_error": 0.19929562370165232,
313 "fourier_reconstruction_error": 0.18154581981703843,
314 "prediction_rms": 0.6864827871322632
315 },
316 {
317 "seed": 3,
318 "dpss_reconstruction_error": 0.2255786421582673,
319 "fourier_reconstruction_error": 0.210578079503818,
320 "prediction_rms": 0.7499093413352966
321 },
322 {
323 "seed": 4,
324 "dpss_reconstruction_error": 0.2059191288393489,
325 "fourier_reconstruction_error": 0.1945450930599946,
326 "prediction_rms": 0.6713075637817383
327 },
328 {
329 "seed": 5,
330 "dpss_reconstruction_error": 0.21693499314985126,
331 "fourier_reconstruction_error": 0.19918181458416848,
332 "prediction_rms": 0.580341637134552
333 },
334 {
335 "seed": 6,
336 "dpss_reconstruction_error": 0.2512841631045155,
337 "fourier_reconstruction_error": 0.22277636588465038,
338 "prediction_rms": 0.654333770275116
339 },
340 {
341 "seed": 7,
342 "dpss_reconstruction_error": 0.2092036386703743,
343 "fourier_reconstruction_error": 0.1886907918314992,
344 "prediction_rms": 0.7088998556137085
345 }
346 ],
347 "confirmed": false
348 }
349}