Initial-Only Weight Decay with Tail Averaging / super_results.json
Audited (legacy)
1{
2 "config": {
3 "dataset": "MNIST train=60000 test=10000",
4 "T": 3000,
5 "width": 128,
6 "batch": 128,
7 "lr": 0.001,
8 "betas": [
9 0.9,
10 0.999
11 ],
12 "weight_decay": 0.05,
13 "p": 0.5,
14 "q": 0.2,
15 "seeds": 5,
16 "device": "cuda",
17 "total_runtime_sec": 461.81064497699845
18 },
19 "operator_check": {
20 "max_E_P2_minus_(1-gamma-lambda)A": -0.00037117758943272516,
21 "max_(1-gamma-lambda)A_minus_scalar_bound": -0.0003762499999999669,
22 "gamma_lambda": 0.13999999999999999,
23 "A_min": 0.8454166666666667,
24 "A_max": 0.8595625,
25 "passed": true
26 },
27 "summary": {
28 "baseline": {
29 "test_accuracy_mean": 0.9772000000000001,
30 "test_accuracy_std": 0.001060660171779825,
31 "best_test_accuracy_mean": 0.9776400000000001,
32 "best_test_accuracy_std": 0.0005813776741499261,
33 "steps_to_best_mean": 2700.0,
34 "steps_to_best_std": 300.0,
35 "final_weight_norm_mean": 15.57228889465332,
36 "final_weight_norm_std": 0.10962862929120072,
37 "mid_gradient_norm_mean": 0.5346320487104357,
38 "mid_gradient_norm_std": 0.010801488519179411,
39 "runtime_sec_mean": 23.057615354601875,
40 "runtime_sec_std": 1.3729107363077253
41 },
42 "initwd": {
43 "test_accuracy_mean": 0.9772399999999999,
44 "test_accuracy_std": 0.0010261578825892063,
45 "best_test_accuracy_mean": 0.9776400000000001,
46 "best_test_accuracy_std": 0.0005856620185738329,
47 "steps_to_best_mean": 2700.0,
48 "steps_to_best_std": 300.0,
49 "final_weight_norm_mean": 15.75581340789795,
50 "final_weight_norm_std": 0.111400605814345,
51 "mid_gradient_norm_mean": 0.5352023165345192,
52 "mid_gradient_norm_std": 0.010488128072847993,
53 "runtime_sec_mean": 21.961071000798256,
54 "runtime_sec_std": 1.3115676939624212
55 },
56 "initwd_tail": {
57 "test_accuracy_mean": 0.97744,
58 "test_accuracy_std": 0.0011013627921806657,
59 "best_test_accuracy_mean": 0.9776400000000001,
60 "best_test_accuracy_std": 0.0005856620185738329,
61 "steps_to_best_mean": 2700.0,
62 "steps_to_best_std": 300.0,
63 "final_weight_norm_mean": 15.75581340789795,
64 "final_weight_norm_std": 0.111400605814345,
65 "mid_gradient_norm_mean": 0.5352023165345192,
66 "mid_gradient_norm_std": 0.010488128072847993,
67 "runtime_sec_mean": 23.616264148801566,
68 "runtime_sec_std": 1.2475728716823775
69 },
70 "wd_tail": {
71 "test_accuracy_mean": 0.97744,
72 "test_accuracy_std": 0.001092245393673077,
73 "best_test_accuracy_mean": 0.9776400000000001,
74 "best_test_accuracy_std": 0.0005813776741499261,
75 "steps_to_best_mean": 2700.0,
76 "steps_to_best_std": 300.0,
77 "final_weight_norm_mean": 15.57228889465332,
78 "final_weight_norm_std": 0.10962862929120072,
79 "mid_gradient_norm_mean": 0.5346320487104357,
80 "mid_gradient_norm_std": 0.010801488519179411,
81 "runtime_sec_mean": 23.652376319398172,
82 "runtime_sec_std": 1.4584476732244598
83 }
84 },
85 "raw": {
86 "baseline": [
87 {
88 "test_accuracy": 0.9779,
89 "last_nonaveraged_accuracy": 0.9779,
90 "best_test_accuracy": 0.9779,
91 "steps_to_best": 3000,
92 "final_weight_norm": 15.454129219055176,
93 "mid_gradient_norm": 0.5471225603520871,
94 "runtime_sec": 22.593704726008582
95 },
96 {
97 "test_accuracy": 0.9782,
98 "last_nonaveraged_accuracy": 0.9782,
99 "best_test_accuracy": 0.9782,
100 "steps_to_best": 3000,
101 "final_weight_norm": 15.6483793258667,
102 "mid_gradient_norm": 0.5196300279647111,
103 "runtime_sec": 21.682183488999726
104 },
105 {
106 "test_accuracy": 0.9778,
107 "last_nonaveraged_accuracy": 0.9778,
108 "best_test_accuracy": 0.9779,
109 "steps_to_best": 2700,
110 "final_weight_norm": 15.522505760192871,
111 "mid_gradient_norm": 0.5286483662202954,
112 "runtime_sec": 21.99832823900215
113 },
114 {
115 "test_accuracy": 0.976,
116 "last_nonaveraged_accuracy": 0.976,
117 "best_test_accuracy": 0.9767,
118 "steps_to_best": 2400,
119 "final_weight_norm": 15.514230728149414,
120 "mid_gradient_norm": 0.5414994262866676,
121 "runtime_sec": 24.74080370500451
122 },
123 {
124 "test_accuracy": 0.9761,
125 "last_nonaveraged_accuracy": 0.9761,
126 "best_test_accuracy": 0.9775,
127 "steps_to_best": 2400,
128 "final_weight_norm": 15.722199440002441,
129 "mid_gradient_norm": 0.536259862728417,
130 "runtime_sec": 24.273056613994413
131 }
132 ],
133 "initwd": [
134 {
135 "test_accuracy": 0.9777,
136 "last_nonaveraged_accuracy": 0.9777,
137 "best_test_accuracy": 0.9778,
138 "steps_to_best": 2700,
139 "final_weight_norm": 15.636313438415527,
140 "mid_gradient_norm": 0.5467983828037977,
141 "runtime_sec": 21.192597489993204
142 },
143 {
144 "test_accuracy": 0.9782,
145 "last_nonaveraged_accuracy": 0.9782,
146 "best_test_accuracy": 0.9782,
147 "steps_to_best": 3000,
148 "final_weight_norm": 15.835013389587402,
149 "mid_gradient_norm": 0.5202843303978443,
150 "runtime_sec": 20.448736153004575
151 },
152 {
153 "test_accuracy": 0.978,
154 "last_nonaveraged_accuracy": 0.978,
155 "best_test_accuracy": 0.978,
156 "steps_to_best": 3000,
157 "final_weight_norm": 15.704302787780762,
158 "mid_gradient_norm": 0.529811853453517,
159 "runtime_sec": 21.56708097200317
160 },
161 {
162 "test_accuracy": 0.9759,
163 "last_nonaveraged_accuracy": 0.9759,
164 "best_test_accuracy": 0.9767,
165 "steps_to_best": 2400,
166 "final_weight_norm": 15.69631576538086,
167 "mid_gradient_norm": 0.5423850215896964,
168 "runtime_sec": 22.930618535989197
169 },
170 {
171 "test_accuracy": 0.9764,
172 "last_nonaveraged_accuracy": 0.9764,
173 "best_test_accuracy": 0.9775,
174 "steps_to_best": 2400,
175 "final_weight_norm": 15.907121658325195,
176 "mid_gradient_norm": 0.5367319944277406,
177 "runtime_sec": 23.666321853001136
178 }
179 ],
180 "initwd_tail": [
181 {
182 "test_accuracy": 0.978,
183 "last_nonaveraged_accuracy": 0.9777,
184 "best_test_accuracy": 0.9778,
185 "steps_to_best": 2700,
186 "final_weight_norm": 15.636313438415527,
187 "mid_gradient_norm": 0.5467983828037977,
188 "runtime_sec": 22.7805057259975
189 },
190 {
191 "test_accuracy": 0.9783,
192 "last_nonaveraged_accuracy": 0.9782,
193 "best_test_accuracy": 0.9782,
194 "steps_to_best": 3000,
195 "final_weight_norm": 15.835013389587402,
196 "mid_gradient_norm": 0.5202843303978443,
197 "runtime_sec": 22.389809077998507
198 },
199 {
200 "test_accuracy": 0.9784,
201 "last_nonaveraged_accuracy": 0.978,
202 "best_test_accuracy": 0.978,
203 "steps_to_best": 3000,
204 "final_weight_norm": 15.704302787780762,
205 "mid_gradient_norm": 0.529811853453517,
206 "runtime_sec": 25.608935714000836
207 },
208 {
209 "test_accuracy": 0.9764,
210 "last_nonaveraged_accuracy": 0.9759,
211 "best_test_accuracy": 0.9767,
212 "steps_to_best": 2400,
213 "final_weight_norm": 15.69631576538086,
214 "mid_gradient_norm": 0.5423850215896964,
215 "runtime_sec": 23.49306345100922
216 },
217 {
218 "test_accuracy": 0.9761,
219 "last_nonaveraged_accuracy": 0.9764,
220 "best_test_accuracy": 0.9775,
221 "steps_to_best": 2400,
222 "final_weight_norm": 15.907121658325195,
223 "mid_gradient_norm": 0.5367319944277406,
224 "runtime_sec": 23.809006775001762
225 }
226 ],
227 "wd_tail": [
228 {
229 "test_accuracy": 0.9781,
230 "last_nonaveraged_accuracy": 0.9779,
231 "best_test_accuracy": 0.9779,
232 "steps_to_best": 3000,
233 "final_weight_norm": 15.454129219055176,
234 "mid_gradient_norm": 0.5471225603520871,
235 "runtime_sec": 21.997487475004164
236 },
237 {
238 "test_accuracy": 0.9784,
239 "last_nonaveraged_accuracy": 0.9782,
240 "best_test_accuracy": 0.9782,
241 "steps_to_best": 3000,
242 "final_weight_norm": 15.6483793258667,
243 "mid_gradient_norm": 0.5196300279647111,
244 "runtime_sec": 22.590078294990235
245 },
246 {
247 "test_accuracy": 0.9782,
248 "last_nonaveraged_accuracy": 0.9778,
249 "best_test_accuracy": 0.9779,
250 "steps_to_best": 2700,
251 "final_weight_norm": 15.522505760192871,
252 "mid_gradient_norm": 0.5286483662202954,
253 "runtime_sec": 25.01615478299209
254 },
255 {
256 "test_accuracy": 0.9762,
257 "last_nonaveraged_accuracy": 0.976,
258 "best_test_accuracy": 0.9767,
259 "steps_to_best": 2400,
260 "final_weight_norm": 15.514230728149414,
261 "mid_gradient_norm": 0.5414994262866676,
262 "runtime_sec": 25.294843284995295
263 },
264 {
265 "test_accuracy": 0.9763,
266 "last_nonaveraged_accuracy": 0.9761,
267 "best_test_accuracy": 0.9775,
268 "steps_to_best": 2400,
269 "final_weight_norm": 15.722199440002441,
270 "mid_gradient_norm": 0.536259862728417,
271 "runtime_sec": 23.363317759009078
272 }
273 ]
274 }
275}