1{
  2  "bench_version": 1,
  3  "track": "tabular",
  4  "model": "mlp_tiny",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "best_cfg": {
  9      "lr": 0.006,
 10      "weight_decay": 0.001
 11    },
 12    "sweep": [
 13      {
 14        "cfg": {
 15          "lr": 0.0015,
 16          "weight_decay": 0.0
 17        },
 18        "mean": 14.94258713722229
 19      },
 20      {
 21        "cfg": {
 22          "lr": 0.0015,
 23          "weight_decay": 0.0001
 24        },
 25        "mean": 14.863693475723267
 26      },
 27      {
 28        "cfg": {
 29          "lr": 0.0015,
 30          "weight_decay": 0.001
 31        },
 32        "mean": 14.837073802947998
 33      },
 34      {
 35        "cfg": {
 36          "lr": 0.003,
 37          "weight_decay": 0.0
 38        },
 39        "mean": 11.12595009803772
 40      },
 41      {
 42        "cfg": {
 43          "lr": 0.003,
 44          "weight_decay": 0.0001
 45        },
 46        "mean": 10.99112582206726
 47      },
 48      {
 49        "cfg": {
 50          "lr": 0.003,
 51          "weight_decay": 0.001
 52        },
 53        "mean": 10.882298469543457
 54      },
 55      {
 56        "cfg": {
 57          "lr": 0.006,
 58          "weight_decay": 0.0
 59        },
 60        "mean": 7.924989342689514
 61      },
 62      {
 63        "cfg": {
 64          "lr": 0.006,
 65          "weight_decay": 0.0001
 66        },
 67        "mean": 7.200156211853027
 68      },
 69      {
 70        "cfg": {
 71          "lr": 0.006,
 72          "weight_decay": 0.001
 73        },
 74        "mean": 7.055913209915161
 75      }
 76    ],
 77    "full": {
 78      "mean": 7.258670151233673,
 79      "std": 0.43432421459398,
 80      "per_seed": [
 81        6.749958515167236,
 82        7.283852577209473,
 83        7.195377826690674,
 84        6.994463920593262,
 85        6.717870235443115,
 86        7.567739486694336,
 87        7.43061637878418,
 88        8.12948226928711
 89      ],
 90      "n": 8
 91    }
 92  },
 93  "idea": {
 94    "mean": 12.942374348640442,
 95    "std": 0.7042793380398409,
 96    "per_seed": [
 97      13.581195831298828,
 98      13.057331085205078,
 99      13.15151309967041,
100      11.582756042480469,
101      12.118292808532715,
102      13.407008171081543,
103      12.833832740783691,
104      13.8070650100708
105    ],
106    "n": 8
107  },
108  "comparison": {
109    "delta_mean": 5.683704197406769,
110    "idea_wins": 0,
111    "n_pairs": 8,
112    "per_seed_diffs": [
113      6.831237316131592,
114      5.7734785079956055,
115      5.956135272979736,
116      4.588292121887207,
117      5.4004225730896,
118      5.839268684387207,
119      5.403216361999512,
120      5.677582740783691
121    ],
122    "p_value": 0.0081,
123    "mde": 0.5271462525241571,
124    "mde_rel_pct": 7.262297935312079,
125    "verdict": "idea worse (significant)",
126    "system_worked": false
127  },
128  "idea_trials": [
129    {
130      "cfg": {
131        "lr": 0.003,
132        "weight_decay": 0.0001,
133        "target_sigma": 2.0
134      },
135      "result": {
136        "mean": 16.349509477615356,
137        "std": 0.9009887062653857,
138        "per_seed": [
139          16.687042236328125,
140          16.01118278503418,
141          16.832225799560547,
142          14.972037315368652,
143          15.328200340270996,
144          16.66720962524414,
145          16.223684310913086,
146          18.074493408203125
147        ],
148        "n": 8
149      }
150    },
151    {
152      "cfg": {
153        "lr": 0.0015,
154        "weight_decay": 0.0001,
155        "target_sigma": 2.0
156      },
157      "result": {
158        "mean": 27.306380033493042,
159        "std": 3.1792541193528128,
160        "per_seed": [
161          27.235408782958984,
162          30.38294792175293,
163          23.430057525634766,
164          23.617746353149414,
165          27.093000411987305,
166          24.0976505279541,
167          30.76529884338379,
168          31.828929901123047
169        ],
170        "n": 8
171      }
172    },
173    {
174      "cfg": {
175        "lr": 0.006,
176        "weight_decay": 0.0001,
177        "target_sigma": 2.0
178      },
179      "result": {
180        "mean": 12.942374348640442,
181        "std": 0.7042793380398409,
182        "per_seed": [
183          13.581195831298828,
184          13.057331085205078,
185          13.15151309967041,
186          11.582756042480469,
187          12.118292808532715,
188          13.407008171081543,
189          12.833832740783691,
190          13.8070650100708
191        ],
192        "n": 8
193      }
194    }
195  ],
196  "protocol_note": "tabular is structurally matched: regularization/stability of linear layers.",
197  "mechanism_signature": {
198    "per_seed": [
199      {
200        "predicted_bound_sigma": [
201          1.9660207790324025,
202          1.9599940777246498,
203          1.7198881439758398
204        ],
205        "observed_true_sigma": [
206          1.965899109840393,
207          1.9599742889404297,
208          1.7198880910873413
209        ],
210        "max_bound_minus_observed": 0.0001216691920093993,
211        "target_sigma": 2.0,
212        "confirmed": true
213      },
214      {
215        "predicted_bound_sigma": [
216          1.987042231331432,
217          1.9654762301286028,
218          1.8110116015778066
219        ],
220        "observed_true_sigma": [
221          1.9857813119888306,
222          1.965454339981079,
223          1.8110116720199585
224        ],
225        "max_bound_minus_observed": 0.0012609193426014187,
226        "target_sigma": 2.0,
227        "confirmed": true
228      },
229      {
230        "predicted_bound_sigma": [
231          1.9618061463008163,
232          1.9643066088926524,
233          1.7638919009485563
234        ],
235        "observed_true_sigma": [
236          1.9616186618804932,
237          1.9643056392669678,
238          1.7638919353485107
239        ],
240        "max_bound_minus_observed": 0.00018748442032312518,
241        "target_sigma": 2.0,
242        "confirmed": true
243      },
244      {
245        "predicted_bound_sigma": [
246          2.0027832775795726,
247          1.9854929319578933,
248          1.7735200867709888
249        ],
250        "observed_true_sigma": [
251          2.00002384185791,
252          1.9853485822677612,
253          1.7735198736190796
254        ],
255        "max_bound_minus_observed": 0.0027594357216624843,
256        "target_sigma": 2.0,
257        "confirmed": true
258      },
259      {
260        "predicted_bound_sigma": [
261          1.9771301980155285,
262          1.9942939741761676,
263          1.7462854647292247
264        ],
265        "observed_true_sigma": [
266          1.9769728183746338,
267          1.9943008422851562,
268          1.7462854385375977
269        ],
270        "max_bound_minus_observed": 0.0001573796408946837,
271        "target_sigma": 2.0,
272        "confirmed": true
273      },
274      {
275        "predicted_bound_sigma": [
276          2.034734505621217,
277          2.0299429975184977,
278          1.743557330086019
279        ],
280        "observed_true_sigma": [
281          2.034123420715332,
282          2.0295772552490234,
283          1.743557095527649
284        ],
285        "max_bound_minus_observed": 0.0006110849058851286,
286        "target_sigma": 2.0,
287        "confirmed": true
288      },
289      {
290        "predicted_bound_sigma": [
291          1.9666434720553738,
292          1.9534572174409695,
293          1.7779871345230767
294        ],
295        "observed_true_sigma": [
296          1.9653239250183105,
297          1.9534378051757812,
298          1.7779872417449951
299        ],
300        "max_bound_minus_observed": 0.0013195470370632734,
301        "target_sigma": 2.0,
302        "confirmed": true
303      },
304      {
305        "predicted_bound_sigma": [
306          2.0642188408904656,
307          2.0205151771620473,
308          1.6440712816243759
309        ],
310        "observed_true_sigma": [
311          2.064063787460327,
312          2.020517587661743,
313          1.6440712213516235
314        ],
315        "max_bound_minus_observed": 0.00015505343013844097,
316        "target_sigma": 2.0,
317        "confirmed": true
318      }
319    ],
320    "predicted_bound_sigma_mean": 1.9089196504193406,
321    "observed_true_sigma_mean": 1.9086223244667053,
322    "max_bound_minus_observed": 0.0027594357216624843,
323    "confirmed": true
324  },
325  "math_check": {
326    "max_upper_bound_violation": 0.0,
327    "mean_bound_gap_squared": 1.9329186102939733,
328    "clustered_recovery_abs_error": 0.0,
329    "passed": true
330  }
331}