1{
  2  "bench_version": 1,
  3  "track": "uncertainty_missingness_classification",
  4  "model": "mlp_tiny",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "full": {
  9      "per_seed": [
 10        0.25932323932647705,
 11        0.32002052664756775,
 12        0.2647111713886261,
 13        0.27379462122917175,
 14        0.334913432598114,
 15        0.27944448590278625,
 16        0.39740344882011414,
 17        0.32050931453704834
 18      ],
 19      "mean": 0.3062650300562382,
 20      "std": 0.04362192624557341,
 21      "n": 8
 22    },
 23    "best_config": {
 24      "lr": 0.01,
 25      "wd": 0.0
 26    },
 27    "sweep": [
 28      {
 29        "config": {
 30          "lr": 0.003,
 31          "wd": 0.0
 32        },
 33        "mean_nll": 0.2802733927965164,
 34        "per_seed": [
 35          {
 36            "nll": 0.2711596190929413,
 37            "err": 0.0949999988079071
 38          },
 39          {
 40            "nll": 0.31240639090538025,
 41            "err": 0.08999999612569809
 42          },
 43          {
 44            "nll": 0.2681650221347809,
 45            "err": 0.10499999672174454
 46          },
 47          {
 48            "nll": 0.26936253905296326,
 49            "err": 0.11249999701976776
 50          }
 51        ]
 52      },
 53      {
 54        "config": {
 55          "lr": 0.003,
 56          "wd": 0.0001
 57        },
 58        "mean_nll": 0.27974314987659454,
 59        "per_seed": [
 60          {
 61            "nll": 0.27134039998054504,
 62            "err": 0.0949999988079071
 63          },
 64          {
 65            "nll": 0.3109743595123291,
 66            "err": 0.08999999612569809
 67          },
 68          {
 69            "nll": 0.2672112286090851,
 70            "err": 0.10249999910593033
 71          },
 72          {
 73            "nll": 0.26944661140441895,
 74            "err": 0.11249999701976776
 75          }
 76        ]
 77      },
 78      {
 79        "config": {
 80          "lr": 0.01,
 81          "wd": 0.0
 82        },
 83        "mean_nll": 0.27946238964796066,
 84        "per_seed": [
 85          {
 86            "nll": 0.25932323932647705,
 87            "err": 0.10499999672174454
 88          },
 89          {
 90            "nll": 0.32002052664756775,
 91            "err": 0.0925000011920929
 92          },
 93          {
 94            "nll": 0.2647111713886261,
 95            "err": 0.10749999433755875
 96          },
 97          {
 98            "nll": 0.27379462122917175,
 99            "err": 0.10749999433755875
100          }
101        ]
102      },
103      {
104        "config": {
105          "lr": 0.01,
106          "wd": 0.0001
107        },
108        "mean_nll": 0.283403217792511,
109        "per_seed": [
110          {
111            "nll": 0.27524203062057495,
112            "err": 0.09999999403953552
113          },
114          {
115            "nll": 0.31805169582366943,
116            "err": 0.08999999612569809
117          },
118          {
119            "nll": 0.26388776302337646,
120            "err": 0.10249999910593033
121          },
122          {
123            "nll": 0.2764313817024231,
124            "err": 0.10749999433755875
125          }
126        ]
127      },
128      {
129        "config": {
130          "lr": 0.03,
131          "wd": 0.0
132        },
133        "mean_nll": 0.30907802283763885,
134        "per_seed": [
135          {
136            "nll": 0.33665135502815247,
137            "err": 0.10999999940395355
138          },
139          {
140            "nll": 0.34054434299468994,
141            "err": 0.08749999850988388
142          },
143          {
144            "nll": 0.25733017921447754,
145            "err": 0.11249999701976776
146          },
147          {
148            "nll": 0.3017862141132355,
149            "err": 0.10749999433755875
150          }
151        ]
152      },
153      {
154        "config": {
155          "lr": 0.03,
156          "wd": 0.0001
157        },
158        "mean_nll": 0.31001827865839005,
159        "per_seed": [
160          {
161            "nll": 0.32909339666366577,
162            "err": 0.10999999940395355
163          },
164          {
165            "nll": 0.34493932127952576,
166            "err": 0.08500000089406967
167          },
168          {
169            "nll": 0.2608874440193176,
170            "err": 0.11249999701976776
171          },
172          {
173            "nll": 0.305152952671051,
174            "err": 0.10999999940395355
175          }
176        ]
177      }
178    ],
179    "full_eval_config": {
180      "lr": 0.01,
181      "wd": 0.0
182    }
183  },
184  "idea": {
185    "per_seed": [
186      0.2529575228691101,
187      0.3246569037437439,
188      0.25593283772468567,
189      0.2860279083251953,
190      0.3488270044326782,
191      0.29746517539024353,
192      0.3761008381843567,
193      0.3177599310874939
194    ],
195    "mean": 0.3074660152196884,
196    "std": 0.04032588351447927,
197    "n": 8,
198    "best_config": {
199      "lr": 0.01,
200      "wd": 0.0
201    },
202    "sweep": [
203      {
204        "config": {
205          "lr": 0.003,
206          "wd": 0.0
207        },
208        "per_seed": [
209          {
210            "nll": 0.27514806389808655,
211            "err": 0.0925000011920929,
212            "missing_corr": 0.9024436840399366,
213            "b": 0.7666630148887634,
214            "pred_missing": 0.44896629452705383,
215            "observed_missing": 0.3125
216          },
217          {
218            "nll": 0.33466601371765137,
219            "err": 0.0925000011920929,
220            "missing_corr": 0.9724430083762355,
221            "b": 0.7647979855537415,
222            "pred_missing": 0.45239901542663574,
223            "observed_missing": 0.3425000011920929
224          },
225          {
226            "nll": 0.26720207929611206,
227            "err": 0.10249999910593033,
228            "missing_corr": 0.9400309045157215,
229            "b": 0.7660092115402222,
230            "pred_missing": 0.4616314172744751,
231            "observed_missing": 0.33500000834465027
232          },
233          {
234            "nll": 0.285520076751709,
235            "err": 0.11749999970197678,
236            "missing_corr": 0.9499087957126232,
237            "b": 0.7629536986351013,
238            "pred_missing": 0.4592474699020386,
239            "observed_missing": 0.3174999952316284
240          },
241          {
242            "nll": 0.3349361717700958,
243            "err": 0.1224999949336052,
244            "missing_corr": 0.9228905173160256,
245            "b": 0.7681577205657959,
246            "pred_missing": 0.4607542157173157,
247            "observed_missing": 0.3425000011920929
248          },
249          {
250            "nll": 0.30076807737350464,
251            "err": 0.11499999463558197,
252            "missing_corr": 0.9735755481574243,
253            "b": 0.7658361196517944,
254            "pred_missing": 0.4581686854362488,
255            "observed_missing": 0.33250001072883606
256          },
257          {
258            "nll": 0.34624627232551575,
259            "err": 0.14000000059604645,
260            "missing_corr": 0.8828189879427482,
261            "b": 0.7663794159889221,
262            "pred_missing": 0.4717622399330139,
263            "observed_missing": 0.3725000023841858
264          },
265          {
266            "nll": 0.3216817080974579,
267            "err": 0.14499999582767487,
268            "missing_corr": 0.9267386520612199,
269            "b": 0.7663792967796326,
270            "pred_missing": 0.46759405732154846,
271            "observed_missing": 0.3425000011920929
272          }
273        ],
274        "mean_nll": 0.30827105790376663
275      },
276      {
277        "config": {
278          "lr": 0.01,
279          "wd": 0.0
280        },
281        "per_seed": [
282          {
283            "nll": 0.2529575228691101,
284            "err": 0.09749999642372131,
285            "missing_corr": 0.8599101025078111,
286            "b": 0.7472167015075684,
287            "pred_missing": 0.4164726138114929,
288            "observed_missing": 0.3125
289          },
290          {
291            "nll": 0.3246569037437439,
292            "err": 0.08999999612569809,
293            "missing_corr": 0.9487513902968302,
294            "b": 0.745702862739563,
295            "pred_missing": 0.41046252846717834,
296            "observed_missing": 0.3425000011920929
297          },
298          {
299            "nll": 0.25593283772468567,
300            "err": 0.11249999701976776,
301            "missing_corr": 0.9430598899116818,
302            "b": 0.7479268908500671,
303            "pred_missing": 0.4195019602775574,
304            "observed_missing": 0.33500000834465027
305          },
306          {
307            "nll": 0.2860279083251953,
308            "err": 0.11499999463558197,
309            "missing_corr": 0.9281673470315555,
310            "b": 0.7177038192749023,
311            "pred_missing": 0.4063892066478729,
312            "observed_missing": 0.3174999952316284
313          },
314          {
315            "nll": 0.3488270044326782,
316            "err": 0.125,
317            "missing_corr": 0.8866245947416767,
318            "b": 0.7491956949234009,
319            "pred_missing": 0.4097135066986084,
320            "observed_missing": 0.3425000011920929
321          },
322          {
323            "nll": 0.29746517539024353,
324            "err": 0.11749999970197678,
325            "missing_corr": 0.9490765968164979,
326            "b": 0.7454859614372253,
327            "pred_missing": 0.41396957635879517,
328            "observed_missing": 0.33250001072883606
329          },
330          {
331            "nll": 0.3761008381843567,
332            "err": 0.1525000035762787,
333            "missing_corr": 0.8037557854015652,
334            "b": 0.7294673919677734,
335            "pred_missing": 0.4210052490234375,
336            "observed_missing": 0.3725000023841858
337          },
338          {
339            "nll": 0.3177599310874939,
340            "err": 0.14000000059604645,
341            "missing_corr": 0.9021314944562635,
342            "b": 0.7316409945487976,
343            "pred_missing": 0.41623786091804504,
344            "observed_missing": 0.3425000011920929
345          }
346        ],
347        "mean_nll": 0.3074660152196884
348      },
349      {
350        "config": {
351          "lr": 0.03,
352          "wd": 0.0
353        },
354        "per_seed": [
355          {
356            "nll": 0.2912931442260742,
357            "err": 0.10249999910593033,
358            "missing_corr": 0.8220934083194001,
359            "b": 0.790917158126831,
360            "pred_missing": 0.3713911473751068,
361            "observed_missing": 0.3125
362          },
363          {
364            "nll": 0.35071563720703125,
365            "err": 0.08249999582767487,
366            "missing_corr": 0.9079289341663003,
367            "b": 0.8076502680778503,
368            "pred_missing": 0.33232399821281433,
369            "observed_missing": 0.3425000011920929
370          },
371          {
372            "nll": 0.24430160224437714,
373            "err": 0.10999999940395355,
374            "missing_corr": 0.9119544553233667,
375            "b": 0.8473079800605774,
376            "pred_missing": 0.3684086501598358,
377            "observed_missing": 0.33500000834465027
378          },
379          {
380            "nll": 0.29751330614089966,
381            "err": 0.11249999701976776,
382            "missing_corr": 0.8516922311869071,
383            "b": 0.7000229358673096,
384            "pred_missing": 0.3121287524700165,
385            "observed_missing": 0.3174999952316284
386          },
387          {
388            "nll": 0.35454967617988586,
389            "err": 0.12999999523162842,
390            "missing_corr": 0.9089462931313597,
391            "b": 0.8252794742584229,
392            "pred_missing": 0.32259058952331543,
393            "observed_missing": 0.3425000011920929
394          },
395          {
396            "nll": 0.28305864334106445,
397            "err": 0.11999999731779099,
398            "missing_corr": 0.9016399678001666,
399            "b": 0.8164690732955933,
400            "pred_missing": 0.3561153709888458,
401            "observed_missing": 0.33250001072883606
402          },
403          {
404            "nll": 0.47652503848075867,
405            "err": 0.17749999463558197,
406            "missing_corr": 0.7801492316430328,
407            "b": 0.7549465298652649,
408            "pred_missing": 0.3328292965888977,
409            "observed_missing": 0.3725000023841858
410          },
411          {
412            "nll": 0.32324108481407166,
413            "err": 0.13499999046325684,
414            "missing_corr": 0.878917930163105,
415            "b": 0.7771673798561096,
416            "pred_missing": 0.3450394868850708,
417            "observed_missing": 0.3425000011920929
418          }
419        ],
420        "mean_nll": 0.32764976657927036
421      }
422    ]
423  },
424  "comparison": {
425    "delta_mean": 0.001200985163450241,
426    "idea_wins": 4,
427    "n_pairs": 8,
428    "per_seed_diffs": [
429      -0.006365716457366943,
430      0.0046363770961761475,
431      -0.00877833366394043,
432      0.01223328709602356,
433      0.013913571834564209,
434      0.018020689487457275,
435      -0.021302610635757446,
436      -0.0027493834495544434
437    ],
438    "p_value": 0.79055,
439    "mde": 0.01120430419306732,
440    "mde_rel_pct": 3.658368763488904,
441    "verdict": "no measurable effect",
442    "system_worked": false
443  },
444  "custom_track": {
445    "name": "uncertainty_missingness_classification",
446    "file": "missingness_bench.py",
447    "domain": "label_missingness"
448  },
449  "math_check": {
450    "a_grad": -0.0441457297454092,
451    "a_expected": -0.0441457297454092,
452    "b_grad": 0.034629446878288096,
453    "b_expected": 0.03462944687828812,
454    "entropy_grad_norm": 0.10446851953879388,
455    "gradient_match": true
456  },
457  "protocol": {
458    "baseline_grid": [
459      {
460        "lr": 0.003,
461        "wd": 0.0
462      },
463      {
464        "lr": 0.003,
465        "wd": 0.0001
466      },
467      {
468        "lr": 0.01,
469        "wd": 0.0
470      },
471      {
472        "lr": 0.01,
473        "wd": 0.0001
474      },
475      {
476        "lr": 0.03,
477        "wd": 0.0
478      },
479      {
480        "lr": 0.03,
481        "wd": 0.0001
482      }
483    ],
484    "idea_grid": [
485      0.003,
486      0.01,
487      0.03
488    ],
489    "paired_deltas_nll": [
490      -0.006365716457366943,
491      0.0046363770961761475,
492      -0.00877833366394043,
493      0.01223328709602356,
494      0.013913571834564209,
495      0.018020689487457275,
496      -0.021302610635757446,
497      -0.0027493834495544434
498    ],
499    "permutation_p": 0.79055,
500    "seeds": [
501      0,
502      1,
503      2,
504      3,
505      4,
506      5,
507      6,
508      7
509    ]
510  },
511  "mechanism_signature": {
512    "predicted": "positive uncertainty-to-missingness slope and high predicted/observed missingness correlation",
513    "observed_b_mean": 0.7392925396561623,
514    "observed_missing_corr_mean": 0.9026846501454853,
515    "confirmed": true
516  }
517}