Utility-Weighted Left-Edge Quantization / bench_stdout.json
Mechanism confirmed, baseline not beaten
1{
2 "bench_version": 1,
3 "track": "tabular",
4 "model": "mlp_tiny",
5 "metric_direction": "lower is better",
6 "n_seeds": 8,
7 "baseline": {
8 "sweep": {
9 "best_cfg": {
10 "lr": 0.01
11 },
12 "sweep": [
13 {
14 "cfg": {
15 "lr": 0.001
16 },
17 "mean": 32.33934259414673
18 },
19 {
20 "cfg": {
21 "lr": 0.003
22 },
23 "mean": 16.37940740585327
24 },
25 {
26 "cfg": {
27 "lr": 0.01
28 },
29 "mean": 10.050109148025513
30 }
31 ],
32 "full": {
33 "mean": 10.512850999832153,
34 "std": 1.6350143489482907,
35 "per_seed": [
36 8.558663368225098,
37 10.783540725708008,
38 9.733381271362305,
39 11.12485122680664,
40 8.562338829040527,
41 12.776362419128418,
42 9.492801666259766,
43 13.070868492126465
44 ],
45 "n": 8
46 }
47 },
48 "best_config": {
49 "lr": 0.01
50 },
51 "full": {
52 "per_seed": [
53 8.558663368225098,
54 10.783540725708008,
55 9.733381271362305,
56 11.12485122680664,
57 8.562338829040527,
58 12.776362419128418,
59 9.492801666259766,
60 13.070868492126465
61 ],
62 "aux": [
63 {
64 "upward_fraction": 0.5784375071525574,
65 "mean_left_gap": 0.009560184553265572,
66 "codes": 16
67 },
68 {
69 "upward_fraction": 0.5663281083106995,
70 "mean_left_gap": 0.03742143139243126,
71 "codes": 16
72 },
73 {
74 "upward_fraction": 0.5204687118530273,
75 "mean_left_gap": 0.07099631428718567,
76 "codes": 16
77 },
78 {
79 "upward_fraction": 0.5039452910423279,
80 "mean_left_gap": 0.08094017207622528,
81 "codes": 16
82 },
83 {
84 "upward_fraction": 0.5638671517372131,
85 "mean_left_gap": 0.05500025674700737,
86 "codes": 16
87 },
88 {
89 "upward_fraction": 0.19499999284744263,
90 "mean_left_gap": 0.03867589682340622,
91 "codes": 16
92 },
93 {
94 "upward_fraction": 0.3338281214237213,
95 "mean_left_gap": 0.015087975189089775,
96 "codes": 16
97 },
98 {
99 "upward_fraction": 0.2247656136751175,
100 "mean_left_gap": 0.09521723538637161,
101 "codes": 16
102 }
103 ],
104 "mean": 10.512850999832153
105 }
106 },
107 "idea": {
108 "config": {
109 "lr": 0.01
110 },
111 "per_seed": [
112 9.773167610168457,
113 11.56466007232666,
114 13.178303718566895,
115 14.895204544067383,
116 10.585610389709473,
117 15.57695484161377,
118 10.29378890991211,
119 14.999134063720703
120 ],
121 "aux": [
122 {
123 "upward_fraction": 0.0,
124 "mean_left_gap": 0.09891153126955032,
125 "codes": 16
126 },
127 {
128 "upward_fraction": 0.0,
129 "mean_left_gap": 0.1319166123867035,
130 "codes": 16
131 },
132 {
133 "upward_fraction": 0.0,
134 "mean_left_gap": 0.20265933871269226,
135 "codes": 16
136 },
137 {
138 "upward_fraction": 0.0,
139 "mean_left_gap": 0.20551525056362152,
140 "codes": 16
141 },
142 {
143 "upward_fraction": 0.0,
144 "mean_left_gap": 0.1258753538131714,
145 "codes": 16
146 },
147 {
148 "upward_fraction": 0.0,
149 "mean_left_gap": 0.14724649488925934,
150 "codes": 16
151 },
152 {
153 "upward_fraction": 0.0,
154 "mean_left_gap": 0.09176831692457199,
155 "codes": 16
156 },
157 {
158 "upward_fraction": 0.0,
159 "mean_left_gap": 0.2208053320646286,
160 "codes": 16
161 }
162 ],
163 "mean": 12.608353018760681,
164 "sweep": [
165 {
166 "lr": 0.003,
167 "per_seed": [
168 17.26456069946289,
169 16.607093811035156,
170 19.64568328857422,
171 18.041486740112305,
172 17.236543655395508,
173 19.011493682861328,
174 15.007841110229492,
175 20.771881103515625
176 ],
177 "aux": [
178 {
179 "upward_fraction": 0.0,
180 "mean_left_gap": 0.11044175922870636,
181 "codes": 16
182 },
183 {
184 "upward_fraction": 0.0,
185 "mean_left_gap": 0.13866443932056427,
186 "codes": 16
187 },
188 {
189 "upward_fraction": 0.0,
190 "mean_left_gap": 0.18446868658065796,
191 "codes": 16
192 },
193 {
194 "upward_fraction": 0.0,
195 "mean_left_gap": 0.20145590603351593,
196 "codes": 16
197 },
198 {
199 "upward_fraction": 0.0,
200 "mean_left_gap": 0.14068220555782318,
201 "codes": 16
202 },
203 {
204 "upward_fraction": 0.0,
205 "mean_left_gap": 0.13724036514759064,
206 "codes": 16
207 },
208 {
209 "upward_fraction": 0.0,
210 "mean_left_gap": 0.10448167473077774,
211 "codes": 16
212 },
213 {
214 "upward_fraction": 0.0,
215 "mean_left_gap": 0.22862808406352997,
216 "codes": 16
217 }
218 ],
219 "mean": 17.948323011398315
220 },
221 {
222 "lr": 0.001,
223 "per_seed": [
224 27.97640609741211,
225 42.11697006225586,
226 70.37665557861328,
227 48.89053726196289,
228 47.3976936340332,
229 83.98968505859375,
230 23.241455078125,
231 54.805789947509766
232 ],
233 "aux": [
234 {
235 "upward_fraction": 0.0,
236 "mean_left_gap": 0.09334667772054672,
237 "codes": 16
238 },
239 {
240 "upward_fraction": 0.0,
241 "mean_left_gap": 0.1060170829296112,
242 "codes": 16
243 },
244 {
245 "upward_fraction": 0.0,
246 "mean_left_gap": 0.10366649180650711,
247 "codes": 16
248 },
249 {
250 "upward_fraction": 0.0,
251 "mean_left_gap": 0.13664911687374115,
252 "codes": 16
253 },
254 {
255 "upward_fraction": 0.0,
256 "mean_left_gap": 0.09772422909736633,
257 "codes": 16
258 },
259 {
260 "upward_fraction": 0.0,
261 "mean_left_gap": 0.07773704826831818,
262 "codes": 16
263 },
264 {
265 "upward_fraction": 0.0,
266 "mean_left_gap": 0.09394310414791107,
267 "codes": 16
268 },
269 {
270 "upward_fraction": 0.0,
271 "mean_left_gap": 0.15413281321525574,
272 "codes": 16
273 }
274 ],
275 "mean": 49.84939908981323
276 },
277 {
278 "lr": 0.01,
279 "per_seed": [
280 9.773167610168457,
281 11.56466007232666,
282 13.178303718566895,
283 14.895204544067383,
284 10.585610389709473,
285 15.57695484161377,
286 10.29378890991211,
287 14.999134063720703
288 ],
289 "aux": [
290 {
291 "upward_fraction": 0.0,
292 "mean_left_gap": 0.09891153126955032,
293 "codes": 16
294 },
295 {
296 "upward_fraction": 0.0,
297 "mean_left_gap": 0.1319166123867035,
298 "codes": 16
299 },
300 {
301 "upward_fraction": 0.0,
302 "mean_left_gap": 0.20265933871269226,
303 "codes": 16
304 },
305 {
306 "upward_fraction": 0.0,
307 "mean_left_gap": 0.20551525056362152,
308 "codes": 16
309 },
310 {
311 "upward_fraction": 0.0,
312 "mean_left_gap": 0.1258753538131714,
313 "codes": 16
314 },
315 {
316 "upward_fraction": 0.0,
317 "mean_left_gap": 0.14724649488925934,
318 "codes": 16
319 },
320 {
321 "upward_fraction": 0.0,
322 "mean_left_gap": 0.09176831692457199,
323 "codes": 16
324 },
325 {
326 "upward_fraction": 0.0,
327 "mean_left_gap": 0.2208053320646286,
328 "codes": 16
329 }
330 ],
331 "mean": 12.608353018760681
332 }
333 ]
334 },
335 "comparison": {
336 "delta_mean": 2.095502018928528,
337 "idea_wins": 0,
338 "n_pairs": 8,
339 "per_seed_diffs": [
340 1.2145042419433594,
341 0.7811193466186523,
342 3.44492244720459,
343 3.770353317260742,
344 2.0232715606689453,
345 2.8005924224853516,
346 0.8009872436523438,
347 1.9282655715942383
348 ],
349 "p_value": 0.0081,
350 "mde": 0.9656413775288165,
351 "mde_rel_pct": 9.185342563537082,
352 "verdict": "idea worse (significant)",
353 "system_worked": false
354 },
355 "mechanism_signature": {
356 "predicted": {
357 "upward_fraction": 0.0,
358 "left_gap_nonnegative": true
359 },
360 "observed": {
361 "mean_upward_fraction": 0.0,
362 "mean_left_gap": 0.15308727882802486
363 },
364 "confirmed": true
365 }
366}