1{
  2  "bench_version": 1,
  3  "track": "tabular",
  4  "model": "mlp_tiny",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "best_cfg": {
  9      "lr": 0.01
 10    },
 11    "sweep": [
 12      {
 13        "cfg": {
 14          "lr": 0.001
 15        },
 16        "mean": 18.263182520866394
 17      },
 18      {
 19        "cfg": {
 20          "lr": 0.003
 21        },
 22        "mean": 13.55293893814087
 23      },
 24      {
 25        "cfg": {
 26          "lr": 0.01
 27        },
 28        "mean": 7.931941092014313
 29      }
 30    ],
 31    "full": {
 32      "mean": 7.931941092014313,
 33      "std": 0.8101161551333251,
 34      "per_seed": [
 35        7.2100510597229,
 36        8.814193725585938,
 37        7.362153053283691,
 38        7.630332946777344,
 39        6.520580291748047,
 40        8.472823143005371,
 41        8.71557903289795,
 42        8.729815483093262
 43      ],
 44      "n": 8
 45    }
 46  },
 47  "idea": {
 48    "per_seed": [
 49      122.21617126464844,
 50      161.85421752929688,
 51      105.60208129882812,
 52      147.32037353515625,
 53      176.8717498779297,
 54      125.64606475830078,
 55      179.69190979003906,
 56      218.0858612060547
 57    ],
 58    "mean": 154.66105365753174,
 59    "best_cfg": {
 60      "lr": 0.003
 61    },
 62    "sweep": [
 63      {
 64        "cfg": {
 65          "lr": 0.001
 66        },
 67        "mean": 162.4086561203003,
 68        "per_seed": [
 69          120.88668060302734,
 70          152.7420654296875,
 71          81.68341064453125,
 72          138.7200927734375,
 73          179.13693237304688,
 74          229.2186279296875,
 75          181.1822967529297,
 76          215.6991424560547
 77        ]
 78      },
 79      {
 80        "cfg": {
 81          "lr": 0.003
 82        },
 83        "mean": 154.66105365753174,
 84        "per_seed": [
 85          122.21617126464844,
 86          161.85421752929688,
 87          105.60208129882812,
 88          147.32037353515625,
 89          176.8717498779297,
 90          125.64606475830078,
 91          179.69190979003906,
 92          218.0858612060547
 93        ]
 94      },
 95      {
 96        "cfg": {
 97          "lr": 0.01
 98        },
 99        "mean": 159.42518997192383,
100        "per_seed": [
101          191.97760009765625,
102          190.82362365722656,
103          185.17913818359375,
104          175.83287048339844,
105          172.7498016357422,
106          86.1894302368164,
107          177.15773010253906,
108          95.49132537841797
109        ]
110      }
111    ]
112  },
113  "comparison": {
114    "delta_mean": 146.72911256551743,
115    "idea_wins": 0,
116    "n_pairs": 8,
117    "per_seed_diffs": [
118      115.00612020492554,
119      153.04002380371094,
120      98.23992824554443,
121      139.6900405883789,
122      170.35116958618164,
123      117.17324161529541,
124      170.9763307571411,
125      209.35604572296143
126    ],
127    "p_value": 0.0081,
128    "mde": 30.622178816214802,
129    "mde_rel_pct": 386.0616015800278,
130    "verdict": "idea worse (significant)",
131    "system_worked": false
132  },
133  "mechanism_signature": {
134    "description": "trained-model checkpoint audit signature",
135    "signature": {
136      "predicted": "validation loss can be flat while audited decision drift is nonzero",
137      "observed": [
138        {
139          "max_drift": 3.82235586643219,
140          "val_range": 218.36089038848877,
141          "audits": 20
142        },
143        {
144          "max_drift": 3.554457426071167,
145          "val_range": 212.48634433746338,
146          "audits": 20
147        },
148        {
149          "max_drift": 5.0,
150          "val_range": 216.55184841156006,
151          "audits": 20
152        },
153        {
154          "max_drift": 5.0,
155          "val_range": 200.55087280273438,
156          "audits": 20
157        },
158        {
159          "max_drift": 3.406560003757477,
160          "val_range": 216.53733348846436,
161          "audits": 20
162        },
163        {
164          "max_drift": 3.6925817728042603,
165          "val_range": 209.86744689941406,
166          "audits": 20
167        },
168        {
169          "max_drift": 3.112291097640991,
170          "val_range": 202.15625095367432,
171          "audits": 20
172        },
173        {
174          "max_drift": 5.0,
175          "val_range": 214.54338932037354,
176          "audits": 20
177        }
178      ],
179      "mean_max_drift": 4.073530770838261,
180      "mean_val_range": 211.3817970752716,
181      "confirmed": false
182    }
183  }
184}