Zero-Augmented Double-Scoring / bench_report.json
Mechanism confirmed, baseline not beaten
1{
2 "bench_version": 1,
3 "track": "tabular",
4 "model": "mlp_tiny",
5 "metric_direction": "lower is better",
6 "n_seeds": 8,
7 "baseline": {
8 "best_cfg": {
9 "lr": 0.01,
10 "density": 0.75
11 },
12 "sweep": [
13 {
14 "cfg": {
15 "lr": 0.001,
16 "density": 0.25
17 },
18 "mean": 235.12103271484375
19 },
20 {
21 "cfg": {
22 "lr": 0.001,
23 "density": 0.5
24 },
25 "mean": 234.88482284545898
26 },
27 {
28 "cfg": {
29 "lr": 0.001,
30 "density": 0.75
31 },
32 "mean": 234.4692726135254
33 },
34 {
35 "cfg": {
36 "lr": 0.003,
37 "density": 0.25
38 },
39 "mean": 234.5323600769043
40 },
41 {
42 "cfg": {
43 "lr": 0.003,
44 "density": 0.5
45 },
46 "mean": 233.07298278808594
47 },
48 {
49 "cfg": {
50 "lr": 0.003,
51 "density": 0.75
52 },
53 "mean": 232.68642044067383
54 },
55 {
56 "cfg": {
57 "lr": 0.01,
58 "density": 0.25
59 },
60 "mean": 232.15027618408203
61 },
62 {
63 "cfg": {
64 "lr": 0.01,
65 "density": 0.5
66 },
67 "mean": 227.00447845458984
68 },
69 {
70 "cfg": {
71 "lr": 0.01,
72 "density": 0.75
73 },
74 "mean": 226.18533325195312
75 }
76 ],
77 "full": {
78 "mean": 224.90276336669922,
79 "std": 11.172746066633566,
80 "per_seed": [
81 239.23036193847656,
82 226.6573944091797,
83 226.5115509033203,
84 212.34202575683594,
85 230.5985870361328,
86 209.33729553222656,
87 214.15821838378906,
88 240.3866729736328
89 ],
90 "n": 8
91 }
92 },
93 "idea": {
94 "mean": 225.713041305542,
95 "std": 11.455704812834473,
96 "per_seed": [
97 241.9227752685547,
98 225.7875518798828,
99 227.34036254882812,
100 211.91632080078125,
101 231.61343383789062,
102 209.4794921875,
103 217.06570434570312,
104 240.5786895751953
105 ],
106 "n": 8,
107 "sweep": [
108 {
109 "cfg": {
110 "lr": 0.001,
111 "rho_aug": 0.5
112 },
113 "mean": 234.92945861816406
114 },
115 {
116 "cfg": {
117 "lr": 0.003,
118 "rho_aug": 0.5
119 },
120 "mean": 232.97465896606445
121 },
122 {
123 "cfg": {
124 "lr": 0.01,
125 "rho_aug": 0.5
126 },
127 "mean": 226.74175262451172
128 }
129 ]
130 },
131 "comparison": {
132 "delta_mean": 0.8102779388427734,
133 "idea_wins": 2,
134 "n_pairs": 8,
135 "per_seed_diffs": [
136 2.692413330078125,
137 -0.869842529296875,
138 0.8288116455078125,
139 -0.4257049560546875,
140 1.0148468017578125,
141 0.1421966552734375,
142 2.9074859619140625,
143 0.1920166015625
144 ],
145 "p_value": 0.16705,
146 "mde": 1.1465610155646562,
147 "mde_rel_pct": 0.509802991480017,
148 "verdict": "no measurable effect",
149 "system_worked": false
150 },
151 "mechanism_signature": {
152 "track_choice": "tabular matches pruning/regularization of a frozen MLP; identical MLP and task are used for both systems.",
153 "shared_lr_grid": [
154 0.001,
155 0.003,
156 0.01
157 ],
158 "baseline_density_grid": [
159 0.25,
160 0.5,
161 0.75
162 ],
163 "idea_sweep": [
164 {
165 "cfg": {
166 "lr": 0.001,
167 "rho_aug": 0.5
168 },
169 "mean": 234.92945861816406
170 },
171 {
172 "cfg": {
173 "lr": 0.003,
174 "rho_aug": 0.5
175 },
176 "mean": 232.97465896606445
177 },
178 {
179 "cfg": {
180 "lr": 0.01,
181 "rho_aug": 0.5
182 },
183 "mean": 226.74175262451172
184 }
185 ],
186 "idea_best_cfg": {
187 "lr": 0.01,
188 "rho_aug": 0.5
189 },
190 "signature": {
191 "prediction": "For iid initialized real/dummy scores and fixed Top-K, expected dummy fraction is 0.5 and expected real density is rho_aug=0.5.",
192 "predicted_dummy_fraction": 0.5,
193 "observed_dummy_fraction_mean": 0.5014583333333333,
194 "predicted_real_density": 0.5,
195 "observed_real_density_mean": 0.5034952799479167,
196 "baseline_observed_real_density": 0.75,
197 "confirmed": true
198 }
199 },
200 "bench_report": {
201 "bench_version": 1,
202 "track": "tabular",
203 "model": "mlp_tiny",
204 "metric_direction": "lower is better",
205 "n_seeds": 8,
206 "baseline": {
207 "best_cfg": {
208 "lr": 0.01,
209 "density": 0.75
210 },
211 "sweep": [
212 {
213 "cfg": {
214 "lr": 0.001,
215 "density": 0.25
216 },
217 "mean": 235.12103271484375
218 },
219 {
220 "cfg": {
221 "lr": 0.001,
222 "density": 0.5
223 },
224 "mean": 234.88482284545898
225 },
226 {
227 "cfg": {
228 "lr": 0.001,
229 "density": 0.75
230 },
231 "mean": 234.4692726135254
232 },
233 {
234 "cfg": {
235 "lr": 0.003,
236 "density": 0.25
237 },
238 "mean": 234.5323600769043
239 },
240 {
241 "cfg": {
242 "lr": 0.003,
243 "density": 0.5
244 },
245 "mean": 233.07298278808594
246 },
247 {
248 "cfg": {
249 "lr": 0.003,
250 "density": 0.75
251 },
252 "mean": 232.68642044067383
253 },
254 {
255 "cfg": {
256 "lr": 0.01,
257 "density": 0.25
258 },
259 "mean": 232.15027618408203
260 },
261 {
262 "cfg": {
263 "lr": 0.01,
264 "density": 0.5
265 },
266 "mean": 227.00447845458984
267 },
268 {
269 "cfg": {
270 "lr": 0.01,
271 "density": 0.75
272 },
273 "mean": 226.18533325195312
274 }
275 ],
276 "full": {
277 "mean": 224.90276336669922,
278 "std": 11.172746066633566,
279 "per_seed": [
280 239.23036193847656,
281 226.6573944091797,
282 226.5115509033203,
283 212.34202575683594,
284 230.5985870361328,
285 209.33729553222656,
286 214.15821838378906,
287 240.3866729736328
288 ],
289 "n": 8
290 }
291 },
292 "idea": {
293 "mean": 225.713041305542,
294 "std": 11.455704812834473,
295 "per_seed": [
296 241.9227752685547,
297 225.7875518798828,
298 227.34036254882812,
299 211.91632080078125,
300 231.61343383789062,
301 209.4794921875,
302 217.06570434570312,
303 240.5786895751953
304 ],
305 "n": 8,
306 "sweep": [
307 {
308 "cfg": {
309 "lr": 0.001,
310 "rho_aug": 0.5
311 },
312 "mean": 234.92945861816406
313 },
314 {
315 "cfg": {
316 "lr": 0.003,
317 "rho_aug": 0.5
318 },
319 "mean": 232.97465896606445
320 },
321 {
322 "cfg": {
323 "lr": 0.01,
324 "rho_aug": 0.5
325 },
326 "mean": 226.74175262451172
327 }
328 ]
329 },
330 "comparison": {
331 "delta_mean": 0.8102779388427734,
332 "idea_wins": 2,
333 "n_pairs": 8,
334 "per_seed_diffs": [
335 2.692413330078125,
336 -0.869842529296875,
337 0.8288116455078125,
338 -0.4257049560546875,
339 1.0148468017578125,
340 0.1421966552734375,
341 2.9074859619140625,
342 0.1920166015625
343 ],
344 "p_value": 0.16705,
345 "mde": 1.1465610155646562,
346 "mde_rel_pct": 0.509802991480017,
347 "verdict": "no measurable effect",
348 "system_worked": false
349 },
350 "mechanism_signature": {
351 "track_choice": "tabular matches pruning/regularization of a frozen MLP; identical MLP and task are used for both systems.",
352 "shared_lr_grid": [
353 0.001,
354 0.003,
355 0.01
356 ],
357 "baseline_density_grid": [
358 0.25,
359 0.5,
360 0.75
361 ],
362 "idea_sweep": [
363 {
364 "cfg": {
365 "lr": 0.001,
366 "rho_aug": 0.5
367 },
368 "mean": 234.92945861816406
369 },
370 {
371 "cfg": {
372 "lr": 0.003,
373 "rho_aug": 0.5
374 },
375 "mean": 232.97465896606445
376 },
377 {
378 "cfg": {
379 "lr": 0.01,
380 "rho_aug": 0.5
381 },
382 "mean": 226.74175262451172
383 }
384 ],
385 "idea_best_cfg": {
386 "lr": 0.01,
387 "rho_aug": 0.5
388 },
389 "signature": {
390 "prediction": "For iid initialized real/dummy scores and fixed Top-K, expected dummy fraction is 0.5 and expected real density is rho_aug=0.5.",
391 "predicted_dummy_fraction": 0.5,
392 "observed_dummy_fraction_mean": 0.5014583333333333,
393 "predicted_real_density": 0.5,
394 "observed_real_density_mean": 0.5034952799479167,
395 "baseline_observed_real_density": 0.75,
396 "confirmed": true
397 }
398 }
399 }
400}