Exponentially Growing Learning Rate with Update-Norm Restarts / results.json
Mechanism confirmed, baseline not beaten
1{
2 "formula_max_abs_error": 5.551115123125783e-17,
3 "scalar_first_restart": [
4 301
5 ],
6 "quadratic_no_restart_final_loss": 1.0035211738579251e+191,
7 "quadratic_restart_final_loss": 7.334216978895374e-12,
8 "quadratic_restart_count": 1,
9 "digits": {
10 "device": "cuda",
11 "fixed": {
12 "final_loss": 0.9185108862982856,
13 "final_acc": 0.8977777777777778,
14 "restarts": 0,
15 "max_lr": 0,
16 "losses": [
17 [
18 1,
19 2.219010959201389,
20 0.30444444444444446
21 ],
22 [
23 2,
24 2.111664021809896,
25 0.5
26 ],
27 [
28 3,
29 2.000046895345052,
30 0.6377777777777778
31 ],
32 [
33 4,
34 1.878652072482639,
35 0.7711111111111111
36 ],
37 [
38 5,
39 1.746902347140842,
40 0.8111111111111111
41 ],
42 [
43 6,
44 1.6081139458550346,
45 0.8377777777777777
46 ],
47 [
48 7,
49 1.4687994723849827,
50 0.8533333333333334
51 ],
52 [
53 8,
54 1.3352897135416666,
55 0.8555555555555555
56 ],
57 [
58 9,
59 1.2123446824815538,
60 0.8622222222222222
61 ],
62 [
63 10,
64 1.1015747409396701,
65 0.8733333333333333
66 ],
67 [
68 11,
69 1.0040602450900609,
70 0.8844444444444445
71 ],
72 [
73 12,
74 0.9185108862982856,
75 0.8977777777777778
76 ]
77 ]
78 },
79 "restart": {
80 "final_loss": 2.145871378580729,
81 "final_acc": 0.43333333333333335,
82 "restarts": 8,
83 "max_lr": 0.014681454416819895,
84 "losses": [
85 [
86 1,
87 2.3220096842447915,
88 0.10222222222222223
89 ],
90 [
91 2,
92 2.3047757297092013,
93 0.11555555555555555
94 ],
95 [
96 3,
97 2.2884149509006075,
98 0.1288888888888889
99 ],
100 [
101 4,
102 2.271092936197917,
103 0.15555555555555556
104 ],
105 [
106 5,
107 2.2555830552842884,
108 0.2
109 ],
110 [
111 6,
112 2.238197462293837,
113 0.24222222222222223
114 ],
115 [
116 7,
117 2.2193995836046008,
118 0.29333333333333333
119 ],
120 [
121 8,
122 2.204880676269531,
123 0.33111111111111113
124 ],
125 [
126 9,
127 2.190496181911892,
128 0.36444444444444446
129 ],
130 [
131 10,
132 2.175982903374566,
133 0.3844444444444444
134 ],
135 [
136 11,
137 2.159806959364149,
138 0.41555555555555557
139 ],
140 [
141 12,
142 2.145871378580729,
143 0.43333333333333335
144 ]
145 ]
146 }
147 }
148}