1{
  2  "bench_version": 1,
  3  "track": "tabular",
  4  "model": "mlp_tiny",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "best_cfg": {
  9      "lr": 0.006,
 10      "weight_decay": 0.0
 11    },
 12    "sweep": [
 13      {
 14        "cfg": {
 15          "lr": 0.0015,
 16          "weight_decay": 0.0
 17        },
 18        "mean": 20.46571445465088
 19      },
 20      {
 21        "cfg": {
 22          "lr": 0.003,
 23          "weight_decay": 0.0
 24        },
 25        "mean": 16.009180068969727
 26      },
 27      {
 28        "cfg": {
 29          "lr": 0.006,
 30          "weight_decay": 0.0
 31        },
 32        "mean": 13.129557490348816
 33      },
 34      {
 35        "cfg": {
 36          "lr": 0.003,
 37          "weight_decay": 0.0001
 38        },
 39        "mean": 15.930055141448975
 40      }
 41    ],
 42    "full": {
 43      "mean": 13.129557490348816,
 44      "std": 1.5250596544234398,
 45      "per_seed": [
 46        13.93191909790039,
 47        13.616814613342285,
 48        10.975268363952637,
 49        13.225879669189453,
 50        10.728046417236328,
 51        12.577923774719238,
 52        14.791933059692383,
 53        15.188674926757812
 54      ],
 55      "n": 8
 56    }
 57  },
 58  "idea": {
 59    "best_cfg": {
 60      "lr": 0.006,
 61      "weight_decay": 0.0
 62    },
 63    "sweep": [
 64      {
 65        "cfg": {
 66          "lr": 0.0015,
 67          "weight_decay": 0.0
 68        },
 69        "mean": 19.026683568954468,
 70        "std": 2.310262868739491,
 71        "per_seed": [
 72          20.679960250854492,
 73          15.556901931762695,
 74          18.93318748474121,
 75          17.145143508911133,
 76          19.445880889892578,
 77          16.939167022705078,
 78          20.18524742126465,
 79          23.327980041503906
 80        ],
 81        "signature_per_seed": [
 82          {
 83            "accepted": 35,
 84            "rejected": 25,
 85            "predicted_decrease_mean": 1.382528269290924,
 86            "observed_decrease_mean": 1.3716121673583985,
 87            "radius_final": 0.012100000000000001
 88          },
 89          {
 90            "accepted": 41,
 91            "rejected": 19,
 92            "predicted_decrease_mean": 2.0615332511866966,
 93            "observed_decrease_mean": 2.022140456408989,
 94            "radius_final": 0.014641000000000003
 95          },
 96          {
 97            "accepted": 40,
 98            "rejected": 20,
 99            "predicted_decrease_mean": 1.1156054258346557,
100            "observed_decrease_mean": 1.1034204244613648,
101            "radius_final": 0.011000000000000001
102          },
103          {
104            "accepted": 41,
105            "rejected": 19,
106            "predicted_decrease_mean": 1.26738692674695,
107            "observed_decrease_mean": 1.255918735411109,
108            "radius_final": 0.011000000000000001
109          },
110          {
111            "accepted": 38,
112            "rejected": 22,
113            "predicted_decrease_mean": 1.2735088334271782,
114            "observed_decrease_mean": 1.2580657758210834,
115            "radius_final": 0.01
116          },
117          {
118            "accepted": 40,
119            "rejected": 20,
120            "predicted_decrease_mean": 1.3169351086020469,
121            "observed_decrease_mean": 1.3040775299072265,
122            "radius_final": 0.01
123          },
124          {
125            "accepted": 41,
126            "rejected": 19,
127            "predicted_decrease_mean": 1.2663456913901538,
128            "observed_decrease_mean": 1.2509966594416921,
129            "radius_final": 0.013310000000000002
130          },
131          {
132            "accepted": 45,
133            "rejected": 15,
134            "predicted_decrease_mean": 1.2946510824892257,
135            "observed_decrease_mean": 1.281169213189019,
136            "radius_final": 0.01806992220000001
137          }
138        ]
139      },
140      {
141        "cfg": {
142          "lr": 0.003,
143          "weight_decay": 0.0
144        },
145        "mean": 15.578829765319824,
146        "std": 1.2037474931922048,
147        "per_seed": [
148          16.088903427124023,
149          13.419285774230957,
150          15.963817596435547,
151          14.6215238571167,
152          14.568916320800781,
153          15.911458969116211,
154          17.396286010742188,
155          16.660446166992188
156        ],
157        "signature_per_seed": [
158          {
159            "accepted": 32,
160            "rejected": 28,
161            "predicted_decrease_mean": 1.2606446049176157,
162            "observed_decrease_mean": 1.1952399015426636,
163            "radius_final": 0.0102
164          },
165          {
166            "accepted": 33,
167            "rejected": 27,
168            "predicted_decrease_mean": 1.7287133065136997,
169            "observed_decrease_mean": 1.6648587313565342,
170            "radius_final": 0.012342000000000002
171          },
172          {
173            "accepted": 38,
174            "rejected": 22,
175            "predicted_decrease_mean": 1.2891257283718962,
176            "observed_decrease_mean": 1.2490642447220652,
177            "radius_final": 0.011000000000000001
178          },
179          {
180            "accepted": 45,
181            "rejected": 15,
182            "predicted_decrease_mean": 1.1980243287152714,
183            "observed_decrease_mean": 1.1577167299058702,
184            "radius_final": 0.011000000000000001
185          },
186          {
187            "accepted": 35,
188            "rejected": 25,
189            "predicted_decrease_mean": 1.508543290410723,
190            "observed_decrease_mean": 1.4751929146902902,
191            "radius_final": 0.01
192          },
193          {
194            "accepted": 35,
195            "rejected": 25,
196            "predicted_decrease_mean": 1.1945056153195246,
197            "observed_decrease_mean": 1.1530200685773577,
198            "radius_final": 0.01
199          },
200          {
201            "accepted": 36,
202            "rejected": 24,
203            "predicted_decrease_mean": 1.228395100062092,
204            "observed_decrease_mean": 1.1819042099846735,
205            "radius_final": 0.01
206          },
207          {
208            "accepted": 43,
209            "rejected": 17,
210            "predicted_decrease_mean": 1.962806150772024,
211            "observed_decrease_mean": 1.9334695505541424,
212            "radius_final": 0.014933820000000006
213          }
214        ]
215      },
216      {
217        "cfg": {
218          "lr": 0.006,
219          "weight_decay": 0.0
220        },
221        "mean": 12.88543701171875,
222        "std": 1.4828934325502452,
223        "per_seed": [
224          13.835183143615723,
225          11.309344291687012,
226          12.560839653015137,
227          12.500975608825684,
228          10.138545989990234,
229          13.836167335510254,
230          14.998884201049805,
231          13.903555870056152
232        ],
233        "signature_per_seed": [
234          {
235            "accepted": 28,
236            "rejected": 32,
237            "predicted_decrease_mean": 1.379367763974837,
238            "observed_decrease_mean": 1.2420953001294817,
239            "radius_final": 0.010404000000000002
240          },
241          {
242            "accepted": 26,
243            "rejected": 34,
244            "predicted_decrease_mean": 1.344854632822367,
245            "observed_decrease_mean": 1.24173248731173,
246            "radius_final": 0.01
247          },
248          {
249            "accepted": 33,
250            "rejected": 27,
251            "predicted_decrease_mean": 1.2900403211965705,
252            "observed_decrease_mean": 1.2163373918244333,
253            "radius_final": 0.011000000000000001
254          },
255          {
256            "accepted": 36,
257            "rejected": 24,
258            "predicted_decrease_mean": 1.089351099398401,
259            "observed_decrease_mean": 1.0336363050672743,
260            "radius_final": 0.011000000000000001
261          },
262          {
263            "accepted": 38,
264            "rejected": 22,
265            "predicted_decrease_mean": 2.0978008941502164,
266            "observed_decrease_mean": 1.8838776789213483,
267            "radius_final": 0.01
268          },
269          {
270            "accepted": 30,
271            "rejected": 30,
272            "predicted_decrease_mean": 1.233123182753722,
273            "observed_decrease_mean": 1.1296147982279459,
274            "radius_final": 0.01
275          },
276          {
277            "accepted": 33,
278            "rejected": 27,
279            "predicted_decrease_mean": 1.392147899577112,
280            "observed_decrease_mean": 1.2645280722415808,
281            "radius_final": 0.01
282          },
283          {
284            "accepted": 39,
285            "rejected": 21,
286            "predicted_decrease_mean": 1.6642315091613011,
287            "observed_decrease_mean": 1.6168656226916192,
288            "radius_final": 0.011220000000000003
289          }
290        ]
291      },
292      {
293        "cfg": {
294          "lr": 0.003,
295          "weight_decay": 0.0001
296        },
297        "mean": 15.578837275505066,
298        "std": 1.203698466172168,
299        "per_seed": [
300          16.08891487121582,
301          13.419283866882324,
302          15.963841438293457,
303          14.621552467346191,
304          14.569123268127441,
305          15.911477088928223,
306          17.396265029907227,
307          16.660240173339844
308        ],
309        "signature_per_seed": [
310          {
311            "accepted": 32,
312            "rejected": 28,
313            "predicted_decrease_mean": 1.2606316241435707,
314            "observed_decrease_mean": 1.1952308416366577,
315            "radius_final": 0.0102
316          },
317          {
318            "accepted": 33,
319            "rejected": 27,
320            "predicted_decrease_mean": 1.7287147830833087,
321            "observed_decrease_mean": 1.6648634708288945,
322            "radius_final": 0.012342000000000002
323          },
324          {
325            "accepted": 38,
326            "rejected": 22,
327            "predicted_decrease_mean": 1.2891209054934352,
328            "observed_decrease_mean": 1.2490592253835577,
329            "radius_final": 0.011000000000000001
330          },
331          {
332            "accepted": 45,
333            "rejected": 15,
334            "predicted_decrease_mean": 1.1980234457386865,
335            "observed_decrease_mean": 1.1577159881591796,
336            "radius_final": 0.011000000000000001
337          },
338          {
339            "accepted": 35,
340            "rejected": 25,
341            "predicted_decrease_mean": 1.5085362383297511,
342            "observed_decrease_mean": 1.475187737601144,
343            "radius_final": 0.01
344          },
345          {
346            "accepted": 35,
347            "rejected": 25,
348            "predicted_decrease_mean": 1.1944936403206416,
349            "observed_decrease_mean": 1.1530044828142438,
350            "radius_final": 0.01
351          },
352          {
353            "accepted": 36,
354            "rejected": 24,
355            "predicted_decrease_mean": 1.228386132667462,
356            "observed_decrease_mean": 1.1818928718566895,
357            "radius_final": 0.01
358          },
359          {
360            "accepted": 43,
361            "rejected": 17,
362            "predicted_decrease_mean": 1.9628009385277712,
363            "observed_decrease_mean": 1.933466112890909,
364            "radius_final": 0.014933820000000006
365          }
366        ]
367      }
368    ],
369    "mean": 12.88543701171875,
370    "std": 1.4828934325502452,
371    "per_seed": [
372      13.835183143615723,
373      11.309344291687012,
374      12.560839653015137,
375      12.500975608825684,
376      10.138545989990234,
377      13.836167335510254,
378      14.998884201049805,
379      13.903555870056152
380    ],
381    "n": 8,
382    "full": {
383      "mean": 12.88543701171875,
384      "std": 1.4828934325502452,
385      "per_seed": [
386        13.835183143615723,
387        11.309344291687012,
388        12.560839653015137,
389        12.500975608825684,
390        10.138545989990234,
391        13.836167335510254,
392        14.998884201049805,
393        13.903555870056152
394      ],
395      "n": 8
396    }
397  },
398  "comparison": {
399    "delta_mean": -0.24412047863006592,
400    "idea_wins": 5,
401    "n_pairs": 8,
402    "per_seed_diffs": [
403      -0.09673595428466797,
404      -2.3074703216552734,
405      1.5855712890625,
406      -0.7249040603637695,
407      -0.5895004272460938,
408      1.2582435607910156,
409      0.20695114135742188,
410      -1.2851190567016602
411    ],
412    "p_value": 0.6102,
413    "mde": 1.0727371153900689,
414    "mde_rel_pct": 8.170398097411958,
415    "verdict": "no significant win",
416    "system_worked": false
417  },
418  "mechanism_signature": {
419    "mechanism_signature": {
420      "predicted_decrease_mean": 1.4363646628793159,
421      "observed_decrease_mean": 1.3285859570519265,
422      "observed_to_predicted_ratio": 0.9261847601497202,
423      "accepted_steps_mean": 32.875,
424      "rejected_steps_mean": 27.125,
425      "confirmed": true
426    },
427    "protocol": {
428      "paired_seeds": [
429        0,
430        1,
431        2,
432        3,
433        4,
434        5,
435        6,
436        7
437      ],
438      "epochs": 15,
439      "train_samples": 400,
440      "test_samples": 200,
441      "grid": [
442        {
443          "lr": 0.0015,
444          "weight_decay": 0.0
445        },
446        {
447          "lr": 0.003,
448          "weight_decay": 0.0
449        },
450        {
451          "lr": 0.006,
452          "weight_decay": 0.0
453        },
454        {
455          "lr": 0.003,
456          "weight_decay": 0.0001
457        }
458      ]
459    },
460    "runtime_sec": 35.358980655670166
461  }
462}