Learning-Rate-Scaled Weight Decay / results.json
Beats tuned baseline
1{
2 "math_verification": {
3 "log_ratio_sweep": [
4 {
5 "r": 0.1,
6 "predicted_log_ratio": 0.1,
7 "observed_log_ratio": 0.09997299621889233,
8 "relative_error": 0.00027003781107676206
9 },
10 {
11 "r": 0.25,
12 "predicted_log_ratio": 0.25,
13 "observed_log_ratio": 0.2498593046380371,
14 "relative_error": 0.0005627814478516013
15 },
16 {
17 "r": 0.5,
18 "predicted_log_ratio": 0.5,
19 "observed_log_ratio": 0.49962443651370425,
20 "relative_error": 0.0007511269725914982
21 },
22 {
23 "r": 0.75,
24 "predicted_log_ratio": 0.75,
25 "observed_log_ratio": 0.749576855177982,
26 "relative_error": 0.0005641930960239941
27 },
28 {
29 "r": 1.0,
30 "predicted_log_ratio": 1.0,
31 "observed_log_ratio": 1.0,
32 "relative_error": 0.0
33 }
34 ],
35 "schedule_cumulative": {
36 "predicted_log_shrinkage": -0.6727382872123923,
37 "observed_log_shrinkage": -0.6727382872123935,
38 "absolute_error": 1.2212453270876722e-15,
39 "constant_log_shrinkage": -0.9050205413161395
40 },
41 "lambda_zero_sweep": [
42 {
43 "r": 0.1,
44 "absolute_difference": 0.0
45 },
46 {
47 "r": 0.5,
48 "absolute_difference": 0.0
49 },
50 {
51 "r": 1.0,
52 "absolute_difference": 0.0
53 }
54 ],
55 "lambda_scaling_sweep": [
56 {
57 "lambda": 0.0,
58 "observed_log_shrinkage": 0.0,
59 "predicted_first_order": -0.0,
60 "normalized_observed": 0.0,
61 "predicted_slope": -3.3564166666666666
62 },
63 {
64 "lambda": 0.02,
65 "observed_log_shrinkage": -0.06714283992963532,
66 "predicted_first_order": -0.06712833333333333,
67 "normalized_observed": -3.357141996481766,
68 "predicted_slope": -3.3564166666666666
69 },
70 {
71 "lambda": 0.05,
72 "observed_log_shrinkage": -0.1679115441609461,
73 "predicted_first_order": -0.16782083333333334,
74 "normalized_observed": -3.3582308832189223,
75 "predicted_slope": -3.3564166666666666
76 },
77 {
78 "lambda": 0.1,
79 "observed_log_shrinkage": -0.33600480778021485,
80 "predicted_first_order": -0.33564166666666667,
81 "normalized_observed": -3.360048077802148,
82 "predicted_slope": -3.3564166666666666
83 },
84 {
85 "lambda": 0.2,
86 "observed_log_shrinkage": -0.6727382872123935,
87 "predicted_first_order": -0.6712833333333333,
88 "normalized_observed": -3.363691436061967,
89 "predicted_slope": -3.3564166666666666
90 },
91 {
92 "lambda": 0.4,
93 "observed_log_shrinkage": -1.348405710433359,
94 "predicted_first_order": -1.3425666666666667,
95 "normalized_observed": -3.3710142760833977,
96 "predicted_slope": -3.3564166666666666
97 }
98 ]
99 },
100 "mini_experiment": {
101 "baseline": {
102 "final_loss": {
103 "mean": 0.058479504038890205,
104 "std": 0.024347165163437805
105 },
106 "loss_at_100": {
107 "mean": 0.7814053297042847,
108 "std": 0.3252069081099237
109 },
110 "loss_at_300": {
111 "mean": 0.07287957891821861,
112 "std": 0.03459615379637844
113 },
114 "final_norm": {
115 "mean": 3.473102251688639,
116 "std": 0.18855447223524782
117 },
118 "norm_at_300": {
119 "mean": 3.4431944688161216,
120 "std": 0.19383601011930413
121 }
122 },
123 "idea": {
124 "final_loss": {
125 "mean": 0.028985902046163876,
126 "std": 0.009098264729836133
127 },
128 "loss_at_100": {
129 "mean": 0.7748445272445679,
130 "std": 0.3224545840341833
131 },
132 "loss_at_300": {
133 "mean": 0.04409672816594442,
134 "std": 0.020935670726080508
135 },
136 "final_norm": {
137 "mean": 3.5797719955444336,
138 "std": 0.1882472925523988
139 },
140 "norm_at_300": {
141 "mean": 3.521018107732137,
142 "std": 0.1946352067112015
143 }
144 },
145 "seeds": [
146 1333,
147 1334,
148 1335
149 ]
150 },
151 "settings": {
152 "steps": 500,
153 "peak_lr": 0.025,
154 "nominal_decay": 0.08,
155 "data": "fixed synthetic linear regression",
156 "seed": 1333
157 }
158}