Weakly Normally Hyperbolic Cyclic Optimizer / bench_report.json
Failed on benchmark
1{
2 "bench_version": 1,
3 "track": "dynamics",
4 "model": "rnn_small",
5 "metric_direction": "lower is better",
6 "n_seeds": 8,
7 "baseline": {
8 "best_cfg": {
9 "lr": 0.0045,
10 "amp": 0.15
11 },
12 "sweep": [
13 {
14 "cfg": {
15 "lr": 0.0015,
16 "amp": 0.15
17 },
18 "mean": 0.0011289938556728885
19 },
20 {
21 "cfg": {
22 "lr": 0.0015,
23 "amp": 0.3
24 },
25 "mean": 0.0011289938556728885
26 },
27 {
28 "cfg": {
29 "lr": 0.0015,
30 "amp": 0.45
31 },
32 "mean": 0.0011289938556728885
33 },
34 {
35 "cfg": {
36 "lr": 0.003,
37 "amp": 0.15
38 },
39 "mean": 0.0007190718897618353
40 },
41 {
42 "cfg": {
43 "lr": 0.003,
44 "amp": 0.3
45 },
46 "mean": 0.0007190718897618353
47 },
48 {
49 "cfg": {
50 "lr": 0.003,
51 "amp": 0.45
52 },
53 "mean": 0.0007190718897618353
54 },
55 {
56 "cfg": {
57 "lr": 0.0045,
58 "amp": 0.15
59 },
60 "mean": 0.0005692571285180748
61 },
62 {
63 "cfg": {
64 "lr": 0.0045,
65 "amp": 0.3
66 },
67 "mean": 0.0005692571285180748
68 },
69 {
70 "cfg": {
71 "lr": 0.0045,
72 "amp": 0.45
73 },
74 "mean": 0.0005692571285180748
75 }
76 ],
77 "full": {
78 "mean": 0.0004823376857530093,
79 "std": 0.00019029999594897276,
80 "per_seed": [
81 0.0004577531653922051,
82 0.00043891146196983755,
83 0.0004620451363734901,
84 0.0009183187503367662,
85 0.000296697486191988,
86 0.0002411661989754066,
87 0.000499835005030036,
88 0.0005439742817543447
89 ],
90 "n": 8
91 }
92 },
93 "idea": {
94 "mean": 0.0007068453487590887,
95 "std": 0.00016523085279349575,
96 "per_seed": [
97 0.0007048308616504073,
98 0.0005210391827858984,
99 0.000925735046621412,
100 0.0009892641101032495,
101 0.0005646041827276349,
102 0.0005265368381515145,
103 0.0006699995719827712,
104 0.0007527529960498214
105 ],
106 "n": 8
107 },
108 "comparison": {
109 "delta_mean": 0.00022450766300607938,
110 "idea_wins": 0,
111 "n_pairs": 8,
112 "per_seed_diffs": [
113 0.0002470776962582022,
114 8.21277208160609e-05,
115 0.00046368991024792194,
116 7.094535976648331e-05,
117 0.0002679066965356469,
118 0.00028537063917610794,
119 0.00017016456695273519,
120 0.00020877871429547668
121 ],
122 "p_value": 0.0081,
123 "mde": 0.00010498848390242133,
124 "mde_rel_pct": 21.76659361345918,
125 "verdict": "idea worse (significant)",
126 "system_worked": false
127 },
128 "mechanism_signature": {
129 "period": 8,
130 "predicted_transverse_multiplier": 0.19839029955701754,
131 "observed_trained_model_multiplier": 0.9998654896163849,
132 "ratio_observed_to_predicted": 5.039891022136507,
133 "confirmed": false,
134 "seed": 0,
135 "probe": "full_batch_gradient_on_trained_rnn",
136 "idea_grid": [
137 {
138 "cfg": {
139 "lr": 0.0015,
140 "amp": 0.15
141 },
142 "result": {
143 "mean": 0.0011780593122239225,
144 "std": 0.00032624222582114425,
145 "per_seed": [
146 0.001007628976367414,
147 0.0009621493518352509,
148 0.00144378119148314,
149 0.001640895614400506,
150 0.0007246441091410816,
151 0.0009201627690345049,
152 0.0010880414629355073,
153 0.001637171022593975
154 ],
155 "n": 8
156 }
157 },
158 {
159 "cfg": {
160 "lr": 0.0015,
161 "amp": 0.3
162 },
163 "result": {
164 "mean": 0.0011592258815653622,
165 "std": 0.0003165982136162725,
166 "per_seed": [
167 0.0010295703541487455,
168 0.0009471527300775051,
169 0.0013883747160434723,
170 0.0016039481852203608,
171 0.0007215833757072687,
172 0.0008863679831847548,
173 0.0010725036263465881,
174 0.0016243060817942023
175 ],
176 "n": 8
177 }
178 },
179 {
180 "cfg": {
181 "lr": 0.0015,
182 "amp": 0.45
183 },
184 "result": {
185 "mean": 0.001143339708505664,
186 "std": 0.0003078421194445482,
187 "per_seed": [
188 0.001056911307387054,
189 0.0009342115954495966,
190 0.0013383073965087533,
191 0.0015667646657675505,
192 0.0007180622196756303,
193 0.0008587845950387418,
194 0.0010603303089737892,
195 0.0016133455792441964
196 ],
197 "n": 8
198 }
199 },
200 {
201 "cfg": {
202 "lr": 0.003,
203 "amp": 0.15
204 },
205 "result": {
206 "mean": 0.0007068453487590887,
207 "std": 0.00016523085279349575,
208 "per_seed": [
209 0.0007048308616504073,
210 0.0005210391827858984,
211 0.000925735046621412,
212 0.0009892641101032495,
213 0.0005646041827276349,
214 0.0005265368381515145,
215 0.0006699995719827712,
216 0.0007527529960498214
217 ],
218 "n": 8
219 }
220 },
221 {
222 "cfg": {
223 "lr": 0.003,
224 "amp": 0.3
225 },
226 "result": {
227 "mean": 0.0007164533963077702,
228 "std": 0.00017887676692651688,
229 "per_seed": [
230 0.0007223670254461467,
231 0.000520958798006177,
232 0.0009306888096034527,
233 0.001032140338793397,
234 0.0005676168366335332,
235 0.0005160922301001847,
236 0.0006507713696919382,
237 0.0007909917621873319
238 ],
239 "n": 8
240 }
241 },
242 {
243 "cfg": {
244 "lr": 0.003,
245 "amp": 0.45
246 },
247 "result": {
248 "mean": 0.0007457361061824486,
249 "std": 0.0001965598982264345,
250 "per_seed": [
251 0.0007489511044695973,
252 0.0005425239796750247,
253 0.0009634686284698546,
254 0.0010984869441017509,
255 0.0005722329951822758,
256 0.0005411882302723825,
257 0.0006400459678843617,
258 0.000858990999404341
259 ],
260 "n": 8
261 }
262 },
263 {
264 "cfg": {
265 "lr": 0.0045,
266 "amp": 0.15
267 },
268 "result": {
269 "mean": 0.0008661484971526079,
270 "std": 0.00018684411923707776,
271 "per_seed": [
272 0.0008908140589483082,
273 0.0006379403057508171,
274 0.0011151200160384178,
275 0.0011663035256788135,
276 0.0006185474921949208,
277 0.0007536062621511519,
278 0.000888270209543407,
279 0.0008585861069150269
280 ],
281 "n": 8
282 }
283 },
284 {
285 "cfg": {
286 "lr": 0.0045,
287 "amp": 0.3
288 },
289 "result": {
290 "mean": 0.0008304821385536343,
291 "std": 0.00018304739075372,
292 "per_seed": [
293 0.000872891687322408,
294 0.000613598502241075,
295 0.0010845665819942951,
296 0.001105351373553276,
297 0.0005852845497429371,
298 0.0006964662461541593,
299 0.000861739506945014,
300 0.0008239586604759097
301 ],
302 "n": 8
303 }
304 },
305 {
306 "cfg": {
307 "lr": 0.0045,
308 "amp": 0.45
309 },
310 "result": {
311 "mean": 0.0008041092150961049,
312 "std": 0.00018305490521590442,
313 "per_seed": [
314 0.0008585874456912279,
315 0.0006087046931497753,
316 0.0010566454147920012,
317 0.0010790202068164945,
318 0.0005509810871444643,
319 0.0006521863397210836,
320 0.0008244491182267666,
321 0.0008022994152270257
322 ],
323 "n": 8
324 }
325 }
326 ],
327 "protocol_note": "8 paired seeds; baseline and idea share lr/amp union; 400/100 samples"
328 }
329}