Learning-Rate-Scaled Weight Decay / results.json

✓✓ Beats tuned baseline

Raw ⬇ ZIP
  1{
  2  "math_verification": {
  3    "log_ratio_sweep": [
  4      {
  5        "r": 0.1,
  6        "predicted_log_ratio": 0.1,
  7        "observed_log_ratio": 0.09997299621889233,
  8        "relative_error": 0.00027003781107676206
  9      },
 10      {
 11        "r": 0.25,
 12        "predicted_log_ratio": 0.25,
 13        "observed_log_ratio": 0.2498593046380371,
 14        "relative_error": 0.0005627814478516013
 15      },
 16      {
 17        "r": 0.5,
 18        "predicted_log_ratio": 0.5,
 19        "observed_log_ratio": 0.49962443651370425,
 20        "relative_error": 0.0007511269725914982
 21      },
 22      {
 23        "r": 0.75,
 24        "predicted_log_ratio": 0.75,
 25        "observed_log_ratio": 0.749576855177982,
 26        "relative_error": 0.0005641930960239941
 27      },
 28      {
 29        "r": 1.0,
 30        "predicted_log_ratio": 1.0,
 31        "observed_log_ratio": 1.0,
 32        "relative_error": 0.0
 33      }
 34    ],
 35    "schedule_cumulative": {
 36      "predicted_log_shrinkage": -0.6727382872123923,
 37      "observed_log_shrinkage": -0.6727382872123935,
 38      "absolute_error": 1.2212453270876722e-15,
 39      "constant_log_shrinkage": -0.9050205413161395
 40    },
 41    "lambda_zero_sweep": [
 42      {
 43        "r": 0.1,
 44        "absolute_difference": 0.0
 45      },
 46      {
 47        "r": 0.5,
 48        "absolute_difference": 0.0
 49      },
 50      {
 51        "r": 1.0,
 52        "absolute_difference": 0.0
 53      }
 54    ],
 55    "lambda_scaling_sweep": [
 56      {
 57        "lambda": 0.0,
 58        "observed_log_shrinkage": 0.0,
 59        "predicted_first_order": -0.0,
 60        "normalized_observed": 0.0,
 61        "predicted_slope": -3.3564166666666666
 62      },
 63      {
 64        "lambda": 0.02,
 65        "observed_log_shrinkage": -0.06714283992963532,
 66        "predicted_first_order": -0.06712833333333333,
 67        "normalized_observed": -3.357141996481766,
 68        "predicted_slope": -3.3564166666666666
 69      },
 70      {
 71        "lambda": 0.05,
 72        "observed_log_shrinkage": -0.1679115441609461,
 73        "predicted_first_order": -0.16782083333333334,
 74        "normalized_observed": -3.3582308832189223,
 75        "predicted_slope": -3.3564166666666666
 76      },
 77      {
 78        "lambda": 0.1,
 79        "observed_log_shrinkage": -0.33600480778021485,
 80        "predicted_first_order": -0.33564166666666667,
 81        "normalized_observed": -3.360048077802148,
 82        "predicted_slope": -3.3564166666666666
 83      },
 84      {
 85        "lambda": 0.2,
 86        "observed_log_shrinkage": -0.6727382872123935,
 87        "predicted_first_order": -0.6712833333333333,
 88        "normalized_observed": -3.363691436061967,
 89        "predicted_slope": -3.3564166666666666
 90      },
 91      {
 92        "lambda": 0.4,
 93        "observed_log_shrinkage": -1.348405710433359,
 94        "predicted_first_order": -1.3425666666666667,
 95        "normalized_observed": -3.3710142760833977,
 96        "predicted_slope": -3.3564166666666666
 97      }
 98    ]
 99  },
100  "mini_experiment": {
101    "baseline": {
102      "final_loss": {
103        "mean": 0.058479504038890205,
104        "std": 0.024347165163437805
105      },
106      "loss_at_100": {
107        "mean": 0.7814053297042847,
108        "std": 0.3252069081099237
109      },
110      "loss_at_300": {
111        "mean": 0.07287957891821861,
112        "std": 0.03459615379637844
113      },
114      "final_norm": {
115        "mean": 3.473102251688639,
116        "std": 0.18855447223524782
117      },
118      "norm_at_300": {
119        "mean": 3.4431944688161216,
120        "std": 0.19383601011930413
121      }
122    },
123    "idea": {
124      "final_loss": {
125        "mean": 0.028985902046163876,
126        "std": 0.009098264729836133
127      },
128      "loss_at_100": {
129        "mean": 0.7748445272445679,
130        "std": 0.3224545840341833
131      },
132      "loss_at_300": {
133        "mean": 0.04409672816594442,
134        "std": 0.020935670726080508
135      },
136      "final_norm": {
137        "mean": 3.5797719955444336,
138        "std": 0.1882472925523988
139      },
140      "norm_at_300": {
141        "mean": 3.521018107732137,
142        "std": 0.1946352067112015
143      }
144    },
145    "seeds": [
146      1333,
147      1334,
148      1335
149    ]
150  },
151  "settings": {
152    "steps": 500,
153    "peak_lr": 0.025,
154    "nominal_decay": 0.08,
155    "data": "fixed synthetic linear regression",
156    "seed": 1333
157  }
158}