Exponentially Growing Learning Rate with Update-Norm Restarts / results.json

Mechanism confirmed, baseline not beaten

Raw ⬇ ZIP
  1{
  2  "formula_max_abs_error": 5.551115123125783e-17,
  3  "scalar_first_restart": [
  4    301
  5  ],
  6  "quadratic_no_restart_final_loss": 1.0035211738579251e+191,
  7  "quadratic_restart_final_loss": 7.334216978895374e-12,
  8  "quadratic_restart_count": 1,
  9  "digits": {
 10    "device": "cuda",
 11    "fixed": {
 12      "final_loss": 0.9185108862982856,
 13      "final_acc": 0.8977777777777778,
 14      "restarts": 0,
 15      "max_lr": 0,
 16      "losses": [
 17        [
 18          1,
 19          2.219010959201389,
 20          0.30444444444444446
 21        ],
 22        [
 23          2,
 24          2.111664021809896,
 25          0.5
 26        ],
 27        [
 28          3,
 29          2.000046895345052,
 30          0.6377777777777778
 31        ],
 32        [
 33          4,
 34          1.878652072482639,
 35          0.7711111111111111
 36        ],
 37        [
 38          5,
 39          1.746902347140842,
 40          0.8111111111111111
 41        ],
 42        [
 43          6,
 44          1.6081139458550346,
 45          0.8377777777777777
 46        ],
 47        [
 48          7,
 49          1.4687994723849827,
 50          0.8533333333333334
 51        ],
 52        [
 53          8,
 54          1.3352897135416666,
 55          0.8555555555555555
 56        ],
 57        [
 58          9,
 59          1.2123446824815538,
 60          0.8622222222222222
 61        ],
 62        [
 63          10,
 64          1.1015747409396701,
 65          0.8733333333333333
 66        ],
 67        [
 68          11,
 69          1.0040602450900609,
 70          0.8844444444444445
 71        ],
 72        [
 73          12,
 74          0.9185108862982856,
 75          0.8977777777777778
 76        ]
 77      ]
 78    },
 79    "restart": {
 80      "final_loss": 2.145871378580729,
 81      "final_acc": 0.43333333333333335,
 82      "restarts": 8,
 83      "max_lr": 0.014681454416819895,
 84      "losses": [
 85        [
 86          1,
 87          2.3220096842447915,
 88          0.10222222222222223
 89        ],
 90        [
 91          2,
 92          2.3047757297092013,
 93          0.11555555555555555
 94        ],
 95        [
 96          3,
 97          2.2884149509006075,
 98          0.1288888888888889
 99        ],
100        [
101          4,
102          2.271092936197917,
103          0.15555555555555556
104        ],
105        [
106          5,
107          2.2555830552842884,
108          0.2
109        ],
110        [
111          6,
112          2.238197462293837,
113          0.24222222222222223
114        ],
115        [
116          7,
117          2.2193995836046008,
118          0.29333333333333333
119        ],
120        [
121          8,
122          2.204880676269531,
123          0.33111111111111113
124        ],
125        [
126          9,
127          2.190496181911892,
128          0.36444444444444446
129        ],
130        [
131          10,
132          2.175982903374566,
133          0.3844444444444444
134        ],
135        [
136          11,
137          2.159806959364149,
138          0.41555555555555557
139        ],
140        [
141          12,
142          2.145871378580729,
143          0.43333333333333335
144        ]
145      ]
146    }
147  }
148}