1{
  2  "bench_version": 1,
  3  "track": "tabular",
  4  "model": "mlp_tiny",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "best_cfg": {
  9      "lr": 0.01,
 10      "ratio": [
 11        100.0,
 12        10.0,
 13        1.0
 14      ]
 15    },
 16    "sweep": [
 17      {
 18        "cfg": {
 19          "lr": 0.001,
 20          "ratio": [
 21            1.0,
 22            1.0,
 23            1.0
 24          ]
 25        },
 26        "mean": 128.7099723815918
 27      },
 28      {
 29        "cfg": {
 30          "lr": 0.001,
 31          "ratio": [
 32            10.0,
 33            3.0,
 34            1.0
 35          ]
 36        },
 37        "mean": 128.4849090576172
 38      },
 39      {
 40        "cfg": {
 41          "lr": 0.001,
 42          "ratio": [
 43            100.0,
 44            10.0,
 45            1.0
 46          ]
 47        },
 48        "mean": 127.46754837036133
 49      },
 50      {
 51        "cfg": {
 52          "lr": 0.003,
 53          "ratio": [
 54            1.0,
 55            1.0,
 56            1.0
 57          ]
 58        },
 59        "mean": 16.73896622657776
 60      },
 61      {
 62        "cfg": {
 63          "lr": 0.003,
 64          "ratio": [
 65            10.0,
 66            3.0,
 67            1.0
 68          ]
 69        },
 70        "mean": 16.98913860321045
 71      },
 72      {
 73        "cfg": {
 74          "lr": 0.003,
 75          "ratio": [
 76            100.0,
 77            10.0,
 78            1.0
 79          ]
 80        },
 81        "mean": 17.95883321762085
 82      },
 83      {
 84        "cfg": {
 85          "lr": 0.01,
 86          "ratio": [
 87            1.0,
 88            1.0,
 89            1.0
 90          ]
 91        },
 92        "mean": 11.914008855819702
 93      },
 94      {
 95        "cfg": {
 96          "lr": 0.01,
 97          "ratio": [
 98            10.0,
 99            3.0,
100            1.0
101          ]
102        },
103        "mean": 11.700665712356567
104      },
105      {
106        "cfg": {
107          "lr": 0.01,
108          "ratio": [
109            100.0,
110            10.0,
111            1.0
112          ]
113        },
114        "mean": 10.906596660614014
115      }
116    ],
117    "full": {
118      "mean": 11.41922926902771,
119      "std": 1.3400197166712482,
120      "per_seed": [
121        11.151049613952637,
122        10.485822677612305,
123        10.90726375579834,
124        11.082250595092773,
125        9.644949913024902,
126        11.029101371765137,
127        12.869707107543945,
128        14.18368911743164
129      ],
130      "n": 8
131    }
132  },
133  "idea": {
134    "mean": 11.052955508232117,
135    "std": 1.160223574491651,
136    "per_seed": [
137      10.566213607788086,
138      10.391647338867188,
139      10.485791206359863,
140      11.461767196655273,
141      9.901324272155762,
142      9.85105037689209,
143      13.145121574401855,
144      12.620728492736816
145    ],
146    "n": 8
147  },
148  "comparison": {
149    "delta_mean": -0.36627376079559326,
150    "idea_wins": 5,
151    "n_pairs": 8,
152    "per_seed_diffs": [
153      -0.5848360061645508,
154      -0.09417533874511719,
155      -0.42147254943847656,
156      0.3795166015625,
157      0.2563743591308594,
158      -1.1780509948730469,
159      0.27541446685791016,
160      -1.5629606246948242
161    ],
162    "p_value": 0.2116,
163    "mde": 0.5972631267868054,
164    "mde_rel_pct": 5.230327833129313,
165    "verdict": "no significant win",
166    "system_worked": false
167  },
168  "mechanism_signature": {
169    "mechanism_signature": {
170      "quantity": "parameter-gradient infinity stationarity residual at refreshes",
171      "predicted_epsilon": 0.02,
172      "observed_mean_residual": Infinity,
173      "positive_margin_frequency": 0.0,
174      "confirmed": false
175    },
176    "math_check": {
177      "predicted_weights": [
178        3.0,
179        1.0
180      ],
181      "observed_weights": [
182        2.999999999,
183        1.000000001
184      ],
185      "residual": 1.000000082740371e-09,
186      "passed": true
187    },
188    "idea_sweep": [
189      {
190        "cfg": {
191          "lr": 0.001,
192          "ratio": [
193            1.0,
194            1.0,
195            1.0
196          ]
197        },
198        "result": {
199          "mean": 127.88450622558594,
200          "std": 14.343906975103058,
201          "per_seed": [
202            142.4267578125,
203            132.91229248046875,
204            121.2231216430664,
205            118.38529205322266,
206            105.10956573486328,
207            127.8440170288086,
208            120.74082946777344,
209            154.43417358398438
210          ],
211          "n": 8
212        }
213      },
214      {
215        "cfg": {
216          "lr": 0.001,
217          "ratio": [
218            10.0,
219            3.0,
220            1.0
221          ]
222        },
223        "result": {
224          "mean": 127.65959358215332,
225          "std": 14.388813821417166,
226          "per_seed": [
227            142.22030639648438,
228            132.71397399902344,
229            120.96118927001953,
230            118.2004623413086,
231            104.7640380859375,
232            127.63045501708984,
233            120.5007553100586,
234            154.2855682373047
235          ],
236          "n": 8
237        }
238      },
239      {
240        "cfg": {
241          "lr": 0.001,
242          "ratio": [
243            100.0,
244            10.0,
245            1.0
246          ]
247        },
248        "result": {
249          "mean": 126.63898754119873,
250          "std": 14.61229098741295,
251          "per_seed": [
252            141.26730346679688,
253            131.85458374023438,
254            119.78008270263672,
255            117.36537170410156,
256            103.09697723388672,
257            126.7028579711914,
258            119.42959594726562,
259            153.61512756347656
260          ],
261          "n": 8
262        }
263      },
264      {
265        "cfg": {
266          "lr": 0.003,
267          "ratio": [
268            1.0,
269            1.0,
270            1.0
271          ]
272        },
273        "result": {
274          "mean": 17.716270089149475,
275          "std": 2.1854981702675587,
276          "per_seed": [
277            17.7928466796875,
278            14.453100204467773,
279            20.243223190307617,
280            15.165732383728027,
281            18.23601722717285,
282            15.786874771118164,
283            19.99270248413086,
284            20.059663772583008
285          ],
286          "n": 8
287        }
288      },
289      {
290        "cfg": {
291          "lr": 0.003,
292          "ratio": [
293            10.0,
294            3.0,
295            1.0
296          ]
297        },
298        "result": {
299          "mean": 17.921056747436523,
300          "std": 2.208602464634322,
301          "per_seed": [
302            18.224416732788086,
303            14.801833152770996,
304            20.551616668701172,
305            15.228721618652344,
306            18.417009353637695,
307            15.783102989196777,
308            20.219045639038086,
309            20.14270782470703
310          ],
311          "n": 8
312        }
313      },
314      {
315        "cfg": {
316          "lr": 0.003,
317          "ratio": [
318            100.0,
319            10.0,
320            1.0
321          ]
322        },
323        "result": {
324          "mean": 18.67002558708191,
325          "std": 2.200020788096735,
326          "per_seed": [
327            19.961009979248047,
328            16.091012954711914,
329            21.057720184326172,
330            15.669567108154297,
331            18.654521942138672,
332            16.20878791809082,
333            21.031444549560547,
334            20.686140060424805
335          ],
336          "n": 8
337        }
338      },
339      {
340        "cfg": {
341          "lr": 0.01,
342          "ratio": [
343            1.0,
344            1.0,
345            1.0
346          ]
347        },
348        "result": {
349          "mean": 11.539970397949219,
350          "std": 1.112516442321061,
351          "per_seed": [
352            10.973492622375488,
353            9.973017692565918,
354            10.890155792236328,
355            11.609981536865234,
356            10.801332473754883,
357            11.49035358428955,
358            13.19100284576416,
359            13.390426635742188
360          ],
361          "n": 8
362        }
363      },
364      {
365        "cfg": {
366          "lr": 0.01,
367          "ratio": [
368            10.0,
369            3.0,
370            1.0
371          ]
372        },
373        "result": {
374          "mean": 11.519728541374207,
375          "std": 0.9986832975547216,
376          "per_seed": [
377            10.72226333618164,
378            10.168411254882812,
379            11.434860229492188,
380            11.341761589050293,
381            11.246848106384277,
382            11.001540184020996,
383            13.17798900604248,
384            13.064154624938965
385          ],
386          "n": 8
387        }
388      },
389      {
390        "cfg": {
391          "lr": 0.01,
392          "ratio": [
393            100.0,
394            10.0,
395            1.0
396          ]
397        },
398        "result": {
399          "mean": 11.052955508232117,
400          "std": 1.160223574491651,
401          "per_seed": [
402            10.566213607788086,
403            10.391647338867188,
404            10.485791206359863,
405            11.461767196655273,
406            9.901324272155762,
407            9.85105037689209,
408            13.145121574401855,
409            12.620728492736816
410          ],
411          "n": 8
412        }
413      }
414    ],
415    "protocol_notes": "Matched tabular Friedman regression and mlp_tiny; baseline and controller share all lr and ratio configurations, Adam, epochs, batch, and data seeds."
416  }
417}