Universal Trust-Region Neural Optimizer / bench_report.json
Failed on benchmark
1{
2 "bench_version": 1,
3 "track": "tabular",
4 "model": "mlp_tiny",
5 "metric_direction": "lower is better",
6 "n_seeds": 8,
7 "baseline": {
8 "best_cfg": {
9 "lr": 0.01,
10 "weight_decay": 0.0001
11 },
12 "sweep": [
13 {
14 "cfg": {
15 "lr": 0.001,
16 "weight_decay": 0.0
17 },
18 "mean": 25.60164165496826
19 },
20 {
21 "cfg": {
22 "lr": 0.001,
23 "weight_decay": 0.0001
24 },
25 "mean": 25.508389472961426
26 },
27 {
28 "cfg": {
29 "lr": 0.003,
30 "weight_decay": 0.0
31 },
32 "mean": 13.860755920410156
33 },
34 {
35 "cfg": {
36 "lr": 0.003,
37 "weight_decay": 0.0001
38 },
39 "mean": 13.839483976364136
40 },
41 {
42 "cfg": {
43 "lr": 0.01,
44 "weight_decay": 0.0
45 },
46 "mean": 8.352817177772522
47 },
48 {
49 "cfg": {
50 "lr": 0.01,
51 "weight_decay": 0.0001
52 },
53 "mean": 8.268931150436401
54 }
55 ],
56 "full": {
57 "mean": 8.206986248493195,
58 "std": 0.7130848431694788,
59 "per_seed": [
60 7.954751968383789,
61 9.628533363342285,
62 7.705244541168213,
63 7.787194728851318,
64 7.160106182098389,
65 8.077467918395996,
66 8.747560501098633,
67 8.595030784606934
68 ],
69 "n": 8
70 }
71 },
72 "idea": {
73 "mean": 15.478003978729248,
74 "std": 1.8150899163881997,
75 "per_seed": [
76 15.246652603149414,
77 15.559816360473633,
78 14.572084426879883,
79 13.438393592834473,
80 12.848532676696777,
81 16.05940055847168,
82 17.402509689331055,
83 18.69664192199707
84 ],
85 "n": 8,
86 "sweep": [
87 {
88 "cfg": {
89 "lr": 0.001,
90 "weight_decay": 0.0001,
91 "delta0": 0.05,
92 "damping": 0.01
93 },
94 "mean": 15.361770868301392,
95 "per_seed": [
96 16.585575103759766,
97 15.630778312683105,
98 15.600324630737305,
99 13.63040542602539
100 ]
101 },
102 {
103 "cfg": {
104 "lr": 0.001,
105 "weight_decay": 0.0001,
106 "delta0": 0.2,
107 "damping": 0.01
108 },
109 "mean": 14.837408542633057,
110 "per_seed": [
111 16.259672164916992,
112 15.100025177001953,
113 15.465311050415039,
114 12.524625778198242
115 ]
116 },
117 {
118 "cfg": {
119 "lr": 0.001,
120 "weight_decay": 0.0001,
121 "delta0": 0.8,
122 "damping": 0.01
123 },
124 "mean": 14.70423674583435,
125 "per_seed": [
126 15.246652603149414,
127 15.559816360473633,
128 14.572084426879883,
129 13.438393592834473
130 ]
131 },
132 {
133 "cfg": {
134 "lr": 0.003,
135 "weight_decay": 0.0001,
136 "delta0": 0.05,
137 "damping": 0.01
138 },
139 "mean": 15.361770868301392,
140 "per_seed": [
141 16.585575103759766,
142 15.630778312683105,
143 15.600324630737305,
144 13.63040542602539
145 ]
146 },
147 {
148 "cfg": {
149 "lr": 0.003,
150 "weight_decay": 0.0001,
151 "delta0": 0.2,
152 "damping": 0.01
153 },
154 "mean": 14.837408542633057,
155 "per_seed": [
156 16.259672164916992,
157 15.100025177001953,
158 15.465311050415039,
159 12.524625778198242
160 ]
161 },
162 {
163 "cfg": {
164 "lr": 0.003,
165 "weight_decay": 0.0001,
166 "delta0": 0.8,
167 "damping": 0.01
168 },
169 "mean": 14.70423674583435,
170 "per_seed": [
171 15.246652603149414,
172 15.559816360473633,
173 14.572084426879883,
174 13.438393592834473
175 ]
176 },
177 {
178 "cfg": {
179 "lr": 0.01,
180 "weight_decay": 0.0001,
181 "delta0": 0.05,
182 "damping": 0.01
183 },
184 "mean": 15.361770868301392,
185 "per_seed": [
186 16.585575103759766,
187 15.630778312683105,
188 15.600324630737305,
189 13.63040542602539
190 ]
191 },
192 {
193 "cfg": {
194 "lr": 0.01,
195 "weight_decay": 0.0001,
196 "delta0": 0.2,
197 "damping": 0.01
198 },
199 "mean": 14.837408542633057,
200 "per_seed": [
201 16.259672164916992,
202 15.100025177001953,
203 15.465311050415039,
204 12.524625778198242
205 ]
206 },
207 {
208 "cfg": {
209 "lr": 0.01,
210 "weight_decay": 0.0001,
211 "delta0": 0.8,
212 "damping": 0.01
213 },
214 "mean": 14.70423674583435,
215 "per_seed": [
216 15.246652603149414,
217 15.559816360473633,
218 14.572084426879883,
219 13.438393592834473
220 ]
221 }
222 ]
223 },
224 "comparison": {
225 "delta_mean": 7.2710177302360535,
226 "idea_wins": 0,
227 "n_pairs": 8,
228 "per_seed_diffs": [
229 7.291900634765625,
230 5.931282997131348,
231 6.86683988571167,
232 5.651198863983154,
233 5.688426494598389,
234 7.981932640075684,
235 8.654949188232422,
236 10.101611137390137
237 ],
238 "p_value": 0.0081,
239 "mde": 1.3228311453724848,
240 "mde_rel_pct": 16.118354598380826,
241 "verdict": "idea worse (significant)",
242 "system_worked": false
243 },
244 "mechanism_signature": {
245 "predicted_decrease_mean": 22.893466198609936,
246 "observed_decrease_mean": -36.19620010587904,
247 "rho_median": 0.37037285272992004,
248 "rho_iqr": [
249 -0.050449262451431096,
250 0.8330098316819403
251 ],
252 "reject_fraction_mean": 0.2777777777777778,
253 "confirmed": false
254 },
255 "protocol_notes": "Tabular is the built-in optimizer track; both systems use identical mlp_tiny, data, epochs, and paired seeds. Baseline is Adam via train_model; idea changes only the training optimizer."
256}