Gaussian-compensated Levy neural noise / bench_report.json
Failed on benchmark
1{
2 "bench_version": 1,
3 "track": "tabular",
4 "model": "mlp_tiny",
5 "metric_direction": "lower is better",
6 "n_seeds": 8,
7 "baseline": {
8 "best_cfg": {
9 "lr": 0.006,
10 "epsilon": 0.03
11 },
12 "sweep": [
13 {
14 "cfg": {
15 "lr": 0.0015,
16 "epsilon": 0.03
17 },
18 "mean": 16.612406492233276
19 },
20 {
21 "cfg": {
22 "lr": 0.0015,
23 "epsilon": 0.06
24 },
25 "mean": 16.47487497329712
26 },
27 {
28 "cfg": {
29 "lr": 0.0015,
30 "epsilon": 0.12
31 },
32 "mean": 16.447417736053467
33 },
34 {
35 "cfg": {
36 "lr": 0.003,
37 "epsilon": 0.03
38 },
39 "mean": 12.79465126991272
40 },
41 {
42 "cfg": {
43 "lr": 0.003,
44 "epsilon": 0.06
45 },
46 "mean": 12.77505874633789
47 },
48 {
49 "cfg": {
50 "lr": 0.003,
51 "epsilon": 0.12
52 },
53 "mean": 12.748817920684814
54 },
55 {
56 "cfg": {
57 "lr": 0.006,
58 "epsilon": 0.03
59 },
60 "mean": 9.08214282989502
61 },
62 {
63 "cfg": {
64 "lr": 0.006,
65 "epsilon": 0.06
66 },
67 "mean": 9.164641618728638
68 },
69 {
70 "cfg": {
71 "lr": 0.006,
72 "epsilon": 0.12
73 },
74 "mean": 9.159956216812134
75 }
76 ],
77 "full": {
78 "mean": 9.39148885011673,
79 "std": 1.1083451358712588,
80 "per_seed": [
81 8.348429679870605,
82 10.594329833984375,
83 8.394119262695312,
84 8.991692543029785,
85 7.703206539154053,
86 9.93536376953125,
87 10.861209869384766,
88 10.303559303283691
89 ],
90 "n": 8
91 }
92 },
93 "idea": {
94 "mean": 9.156296849250793,
95 "std": 1.1388098507230549,
96 "per_seed": [
97 7.9084343910217285,
98 10.600011825561523,
99 8.813246726989746,
100 8.956066131591797,
101 7.3792853355407715,
102 8.759374618530273,
103 10.619342803955078,
104 10.21461296081543
105 ],
106 "n": 8
107 },
108 "comparison": {
109 "delta_mean": -0.23519200086593628,
110 "idea_wins": 6,
111 "n_pairs": 8,
112 "per_seed_diffs": [
113 -0.43999528884887695,
114 0.0056819915771484375,
115 0.4191274642944336,
116 -0.03562641143798828,
117 -0.32392120361328125,
118 -1.1759891510009766,
119 -0.2418670654296875,
120 -0.08894634246826172
121 ],
122 "p_value": 0.18585,
123 "mde": 0.38477736395591483,
124 "mde_rel_pct": 4.097085883790751,
125 "verdict": "no significant win",
126 "system_worked": false
127 },
128 "mechanism_signature": {
129 "track_match": "optimizer noise -> tabular Friedman#1",
130 "mechanism": {
131 "prediction": "compensated small-jump update variance equals scale^2*c*epsilon^(2-alpha)/(2-alpha)",
132 "predicted_update_variance": 4.898979485566358e-08,
133 "observed_update_variance": 4.051713719621018e-08,
134 "relative_error": 0.17294740025787028,
135 "confirmed": false,
136 "trained_model_runs": 8,
137 "selected_cfg": {
138 "lr": 0.006,
139 "epsilon": 0.06
140 },
141 "idea_settings_evaluated": [
142 {
143 "cfg": {
144 "lr": 0.006,
145 "epsilon": 0.06
146 },
147 "result": {
148 "mean": 9.156296849250793,
149 "std": 1.1388098507230549,
150 "per_seed": [
151 7.9084343910217285,
152 10.600011825561523,
153 8.813246726989746,
154 8.956066131591797,
155 7.3792853355407715,
156 8.759374618530273,
157 10.619342803955078,
158 10.21461296081543
159 ],
160 "n": 8
161 }
162 },
163 {
164 "cfg": {
165 "lr": 0.006,
166 "epsilon": 0.03
167 },
168 "result": {
169 "mean": 9.438631236553192,
170 "std": 1.0926888335764746,
171 "per_seed": [
172 8.160052299499512,
173 10.666433334350586,
174 8.840189933776855,
175 8.998281478881836,
176 7.716319561004639,
177 10.017929077148438,
178 10.732819557189941,
179 10.37702465057373
180 ],
181 "n": 8
182 }
183 },
184 {
185 "cfg": {
186 "lr": 0.006,
187 "epsilon": 0.12
188 },
189 "result": {
190 "mean": 9.451428532600403,
191 "std": 1.0498153270226125,
192 "per_seed": [
193 8.208366394042969,
194 10.474081039428711,
195 8.878252983093262,
196 9.130773544311523,
197 7.725558280944824,
198 10.141274452209473,
199 10.64528751373291,
200 10.40783405303955
201 ],
202 "n": 8
203 }
204 }
205 ]
206 }
207 },
208 "idea_sweep": [
209 {
210 "cfg": {
211 "lr": 0.006,
212 "epsilon": 0.06
213 },
214 "result": {
215 "mean": 9.156296849250793,
216 "std": 1.1388098507230549,
217 "per_seed": [
218 7.9084343910217285,
219 10.600011825561523,
220 8.813246726989746,
221 8.956066131591797,
222 7.3792853355407715,
223 8.759374618530273,
224 10.619342803955078,
225 10.21461296081543
226 ],
227 "n": 8
228 }
229 },
230 {
231 "cfg": {
232 "lr": 0.006,
233 "epsilon": 0.03
234 },
235 "result": {
236 "mean": 9.438631236553192,
237 "std": 1.0926888335764746,
238 "per_seed": [
239 8.160052299499512,
240 10.666433334350586,
241 8.840189933776855,
242 8.998281478881836,
243 7.716319561004639,
244 10.017929077148438,
245 10.732819557189941,
246 10.37702465057373
247 ],
248 "n": 8
249 }
250 },
251 {
252 "cfg": {
253 "lr": 0.006,
254 "epsilon": 0.12
255 },
256 "result": {
257 "mean": 9.451428532600403,
258 "std": 1.0498153270226125,
259 "per_seed": [
260 8.208366394042969,
261 10.474081039428711,
262 8.878252983093262,
263 9.130773544311523,
264 7.725558280944824,
265 10.141274452209473,
266 10.64528751373291,
267 10.40783405303955
268 ],
269 "n": 8
270 }
271 }
272 ],
273 "custom_track": null
274}