Exponentially Growing Learning Rate with Update-Norm Restarts / bench_report.json
Mechanism confirmed, baseline not beaten
1{
2 "bench_version": 1,
3 "track": "tabular",
4 "model": "mlp_tiny",
5 "metric_direction": "lower is better",
6 "n_seeds": 8,
7 "baseline": {
8 "sweep": [
9 {
10 "cfg": {
11 "lr": 0.001,
12 "wd": 0.0
13 },
14 "mean": 17.79831099510193
15 },
16 {
17 "cfg": {
18 "lr": 0.001,
19 "wd": 0.0001
20 },
21 "mean": 17.804412841796875
22 },
23 {
24 "cfg": {
25 "lr": 0.003,
26 "wd": 0.0
27 },
28 "mean": 13.153547525405884
29 },
30 {
31 "cfg": {
32 "lr": 0.003,
33 "wd": 0.0001
34 },
35 "mean": 13.136067628860474
36 },
37 {
38 "cfg": {
39 "lr": 0.006,
40 "wd": 0.0
41 },
42 "mean": 9.905518293380737
43 },
44 {
45 "cfg": {
46 "lr": 0.006,
47 "wd": 0.0001
48 },
49 "mean": 9.729963302612305
50 },
51 {
52 "cfg": {
53 "lr": 0.01,
54 "wd": 0.0
55 },
56 "mean": 7.754182696342468
57 },
58 {
59 "cfg": {
60 "lr": 0.01,
61 "wd": 0.0001
62 },
63 "mean": 7.6093244552612305
64 }
65 ],
66 "best_cfg": {
67 "lr": 0.01,
68 "wd": 0.0001
69 },
70 "full": {
71 "per_seed": [
72 7.102456092834473,
73 8.766936302185059,
74 7.349897861480713,
75 7.218007564544678,
76 6.500193119049072,
77 7.548990249633789,
78 8.126578330993652,
79 8.261975288391113
80 ],
81 "mean": 7.609379351139069,
82 "std": 0.6841718521926304,
83 "n": 8,
84 "config": {
85 "lr": 0.01,
86 "wd": 0.0001
87 }
88 },
89 "audit_runs": [
90 {
91 "metric": 7.102456092834473,
92 "history": [
93 214.53998596191406,
94 150.79551391601564,
95 45.06250183105469,
96 44.90724487304688,
97 22.644885025024415,
98 22.263154296875,
99 24.194852905273436,
100 15.885811157226563,
101 17.668506774902344,
102 14.434557914733887,
103 13.434552192687988,
104 12.095326461791991,
105 11.769148025512695,
106 10.334355964660645,
107 10.175729064941406,
108 8.749533672332763,
109 8.571477165222168,
110 7.892980594635009,
111 7.619370031356811,
112 7.219865341186523
113 ]
114 },
115 {
116 "metric": 8.766936302185059,
117 "history": [
118 216.17651611328125,
119 156.86871520996092,
120 45.16012939453125,
121 50.300657958984374,
122 22.345151824951174,
123 21.229163131713868,
124 21.279377746582032,
125 13.878670806884765,
126 16.061290435791015,
127 13.24773666381836,
128 12.390899543762208,
129 11.70387607574463,
130 11.164104766845703,
131 10.362736930847168,
132 10.45274757385254,
133 9.97411792755127,
134 8.932565460205078,
135 8.518637466430665,
136 8.128562622070312,
137 7.766393203735351
138 ]
139 },
140 {
141 "metric": 7.349897861480713,
142 "history": [
143 219.39349243164062,
144 146.39451263427733,
145 36.564744262695314,
146 48.9598583984375,
147 19.486346893310547,
148 27.214415740966796,
149 19.930125961303713,
150 16.0131876373291,
151 18.18076370239258,
152 12.916956748962402,
153 15.148296737670899,
154 11.739884834289551,
155 10.988883819580078,
156 9.74200210571289,
157 9.59509126663208,
158 8.717781295776367,
159 8.149822845458985,
160 7.8048429107666015,
161 7.387167625427246,
162 7.143137340545654
163 ]
164 },
165 {
166 "metric": 7.218007564544678,
167 "history": [
168 215.29245727539063,
169 153.3181286621094,
170 43.44981231689453,
171 51.55850280761719,
172 24.195310096740723,
173 24.33369873046875,
174 29.513503189086915,
175 17.208366165161134,
176 16.743267669677735,
177 16.64546733856201,
178 13.176614532470703,
179 13.314536399841309,
180 12.080739440917968,
181 11.744078483581543,
182 11.132203235626221,
183 10.857590484619141,
184 9.981960220336914,
185 9.732174415588378,
186 9.127348098754883,
187 8.670081977844239
188 ]
189 },
190 {
191 "metric": 6.500193119049072,
192 "history": [
193 232.4084051513672,
194 140.419755859375,
195 30.65204315185547,
196 56.73716537475586,
197 19.858126831054687,
198 31.676787109375,
199 23.139746131896974,
200 17.12270294189453,
201 19.785086517333983,
202 13.713419494628907,
203 15.035507736206055,
204 12.384070320129394,
205 11.58296558380127,
206 10.596111755371094,
207 9.830599327087402,
208 8.959623203277587,
209 8.42355484008789,
210 7.857340164184571,
211 7.7755109977722165,
212 7.369232406616211
213 ]
214 },
215 {
216 "metric": 7.548990249633789,
217 "history": [
218 218.01401733398438,
219 161.49740661621092,
220 52.60572364807129,
221 45.408958740234375,
222 24.290284996032714,
223 19.144558486938475,
224 21.734841079711913,
225 13.96752471923828,
226 13.811888771057129,
227 13.178979797363281,
228 11.018290138244629,
229 11.17254119873047,
230 9.871922531127929,
231 9.451333503723145,
232 8.699445419311523,
233 8.35977834701538,
234 7.6648030090332036,
235 7.595121612548828,
236 7.025888919830322,
237 7.046415634155274
238 ]
239 },
240 {
241 "metric": 8.126578330993652,
242 "history": [
243 219.12206787109375,
244 150.0679965209961,
245 39.91819145202637,
246 53.258799896240234,
247 22.7507950592041,
248 25.290062026977537,
249 24.593091125488282,
250 16.55346389770508,
251 17.977804107666017,
252 14.593553962707519,
253 14.882343139648437,
254 13.11092544555664,
255 12.004822521209716,
256 11.090541763305664,
257 10.585743408203125,
258 9.785175476074219,
259 9.138872356414796,
260 8.626234855651855,
261 8.139089069366456,
262 7.8595320510864255
263 ]
264 },
265 {
266 "metric": 8.261975288391113,
267 "history": [
268 235.50824462890625,
269 181.1517953491211,
270 66.92042526245118,
271 44.197750854492185,
272 29.396741561889648,
273 23.55883369445801,
274 26.864130783081055,
275 17.323555297851563,
276 18.29806915283203,
277 16.233560333251955,
278 14.512285614013672,
279 13.535792064666747,
280 11.563432502746583,
281 11.270505142211913,
282 9.929189453125,
283 9.247753982543946,
284 8.481544513702392,
285 7.953108673095703,
286 7.3793183708190915,
287 7.1775992965698245
288 ]
289 }
290 ]
291 },
292 "idea": {
293 "per_seed": [
294 8.10361099243164,
295 8.1812162399292,
296 7.210653305053711,
297 8.95467758178711,
298 7.580370903015137,
299 8.632294654846191,
300 8.847290992736816,
301 9.434948921203613
302 ],
303 "mean": 8.368132948875427,
304 "std": 0.6934283174375584,
305 "n": 8,
306 "config": {
307 "lr": 0.01,
308 "r": 0.01,
309 "beta": 1.3
310 },
311 "diagnostics": [
312 {
313 "metric": 8.10361099243164,
314 "restarts": 3,
315 "max_lr": 0.017332530178673953,
316 "update_norm_mean": 0.12184708876397417,
317 "update_growth_mean": 1.0122137089134595
318 },
319 {
320 "metric": 8.1812162399292,
321 "restarts": 3,
322 "max_lr": 0.014769807938826427,
323 "update_norm_mean": 0.11243514141108242,
324 "update_growth_mean": 1.0269314387105273
325 },
326 {
327 "metric": 7.210653305053711,
328 "restarts": 3,
329 "max_lr": 0.014769807938826427,
330 "update_norm_mean": 0.11399464353001698,
331 "update_growth_mean": 1.0031604459630117
332 },
333 {
334 "metric": 8.95467758178711,
335 "restarts": 0,
336 "max_lr": 0.02203396426255937,
337 "update_norm_mean": 0.1521656754655411,
338 "update_growth_mean": 1.0103780184537154
339 },
340 {
341 "metric": 7.580370903015137,
342 "restarts": 0,
343 "max_lr": 0.02203396426255937,
344 "update_norm_mean": 0.151965435222015,
345 "update_growth_mean": 1.0102908595171285
346 },
347 {
348 "metric": 8.632294654846191,
349 "restarts": 2,
350 "max_lr": 0.016820276496988865,
351 "update_norm_mean": 0.11849648705603952,
352 "update_growth_mean": 1.0099841345527327
353 },
354 {
355 "metric": 8.847290992736816,
356 "restarts": 4,
357 "max_lr": 0.01648721270700128,
358 "update_norm_mean": 0.1123172825782556,
359 "update_growth_mean": 1.024348321067094
360 },
361 {
362 "metric": 9.434948921203613,
363 "restarts": 4,
364 "max_lr": 0.015219615556186337,
365 "update_norm_mean": 0.11368357661240187,
366 "update_growth_mean": 1.0083068740327106
367 }
368 ]
369 },
370 "comparison": {
371 "delta_mean": 0.7587535977363586,
372 "idea_wins": 2,
373 "n_pairs": 8,
374 "per_seed_diffs": [
375 1.001154899597168,
376 -0.5857200622558594,
377 -0.13924455642700195,
378 1.7366700172424316,
379 1.0801777839660645,
380 1.0833044052124023,
381 0.7207126617431641,
382 1.1729736328125
383 ],
384 "p_value": 0.04105,
385 "mde": 0.632922491434946,
386 "mde_rel_pct": 8.317662482423119,
387 "verdict": "idea worse (significant)",
388 "system_worked": false
389 },
390 "search_space_parity": true,
391 "baseline_knobs_swept": [
392 "lr",
393 "weight_decay"
394 ],
395 "mechanism_signature": {
396 "prediction": "restart criterion detects update growth at least beta*exp(r) schedule growth",
397 "predicted_threshold": 1.3130652172094184,
398 "observed_mean_growth_over_schedule": 1.0031201995404122,
399 "observed_restarts_mean": 2.375,
400 "confirmed": true
401 }
402}