{
  "bench_version": 1,
  "track": "uncertainty_missingness_classification",
  "model": "mlp_tiny",
  "metric_direction": "lower is better",
  "n_seeds": 8,
  "baseline": {
    "full": {
      "per_seed": [
        0.25932323932647705,
        0.32002052664756775,
        0.2647111713886261,
        0.27379462122917175,
        0.334913432598114,
        0.27944448590278625,
        0.39740344882011414,
        0.32050931453704834
      ],
      "mean": 0.3062650300562382,
      "std": 0.04362192624557341,
      "n": 8
    },
    "best_config": {
      "lr": 0.01,
      "wd": 0.0
    },
    "sweep": [
      {
        "config": {
          "lr": 0.003,
          "wd": 0.0
        },
        "mean_nll": 0.2802733927965164,
        "per_seed": [
          {
            "nll": 0.2711596190929413,
            "err": 0.0949999988079071
          },
          {
            "nll": 0.31240639090538025,
            "err": 0.08999999612569809
          },
          {
            "nll": 0.2681650221347809,
            "err": 0.10499999672174454
          },
          {
            "nll": 0.26936253905296326,
            "err": 0.11249999701976776
          }
        ]
      },
      {
        "config": {
          "lr": 0.003,
          "wd": 0.0001
        },
        "mean_nll": 0.27974314987659454,
        "per_seed": [
          {
            "nll": 0.27134039998054504,
            "err": 0.0949999988079071
          },
          {
            "nll": 0.3109743595123291,
            "err": 0.08999999612569809
          },
          {
            "nll": 0.2672112286090851,
            "err": 0.10249999910593033
          },
          {
            "nll": 0.26944661140441895,
            "err": 0.11249999701976776
          }
        ]
      },
      {
        "config": {
          "lr": 0.01,
          "wd": 0.0
        },
        "mean_nll": 0.27946238964796066,
        "per_seed": [
          {
            "nll": 0.25932323932647705,
            "err": 0.10499999672174454
          },
          {
            "nll": 0.32002052664756775,
            "err": 0.0925000011920929
          },
          {
            "nll": 0.2647111713886261,
            "err": 0.10749999433755875
          },
          {
            "nll": 0.27379462122917175,
            "err": 0.10749999433755875
          }
        ]
      },
      {
        "config": {
          "lr": 0.01,
          "wd": 0.0001
        },
        "mean_nll": 0.283403217792511,
        "per_seed": [
          {
            "nll": 0.27524203062057495,
            "err": 0.09999999403953552
          },
          {
            "nll": 0.31805169582366943,
            "err": 0.08999999612569809
          },
          {
            "nll": 0.26388776302337646,
            "err": 0.10249999910593033
          },
          {
            "nll": 0.2764313817024231,
            "err": 0.10749999433755875
          }
        ]
      },
      {
        "config": {
          "lr": 0.03,
          "wd": 0.0
        },
        "mean_nll": 0.30907802283763885,
        "per_seed": [
          {
            "nll": 0.33665135502815247,
            "err": 0.10999999940395355
          },
          {
            "nll": 0.34054434299468994,
            "err": 0.08749999850988388
          },
          {
            "nll": 0.25733017921447754,
            "err": 0.11249999701976776
          },
          {
            "nll": 0.3017862141132355,
            "err": 0.10749999433755875
          }
        ]
      },
      {
        "config": {
          "lr": 0.03,
          "wd": 0.0001
        },
        "mean_nll": 0.31001827865839005,
        "per_seed": [
          {
            "nll": 0.32909339666366577,
            "err": 0.10999999940395355
          },
          {
            "nll": 0.34493932127952576,
            "err": 0.08500000089406967
          },
          {
            "nll": 0.2608874440193176,
            "err": 0.11249999701976776
          },
          {
            "nll": 0.305152952671051,
            "err": 0.10999999940395355
          }
        ]
      }
    ],
    "full_eval_config": {
      "lr": 0.01,
      "wd": 0.0
    }
  },
  "idea": {
    "per_seed": [
      0.2529575228691101,
      0.3246569037437439,
      0.25593283772468567,
      0.2860279083251953,
      0.3488270044326782,
      0.29746517539024353,
      0.3761008381843567,
      0.3177599310874939
    ],
    "mean": 0.3074660152196884,
    "std": 0.04032588351447927,
    "n": 8,
    "best_config": {
      "lr": 0.01,
      "wd": 0.0
    },
    "sweep": [
      {
        "config": {
          "lr": 0.003,
          "wd": 0.0
        },
        "per_seed": [
          {
            "nll": 0.27514806389808655,
            "err": 0.0925000011920929,
            "missing_corr": 0.9024436840399366,
            "b": 0.7666630148887634,
            "pred_missing": 0.44896629452705383,
            "observed_missing": 0.3125
          },
          {
            "nll": 0.33466601371765137,
            "err": 0.0925000011920929,
            "missing_corr": 0.9724430083762355,
            "b": 0.7647979855537415,
            "pred_missing": 0.45239901542663574,
            "observed_missing": 0.3425000011920929
          },
          {
            "nll": 0.26720207929611206,
            "err": 0.10249999910593033,
            "missing_corr": 0.9400309045157215,
            "b": 0.7660092115402222,
            "pred_missing": 0.4616314172744751,
            "observed_missing": 0.33500000834465027
          },
          {
            "nll": 0.285520076751709,
            "err": 0.11749999970197678,
            "missing_corr": 0.9499087957126232,
            "b": 0.7629536986351013,
            "pred_missing": 0.4592474699020386,
            "observed_missing": 0.3174999952316284
          },
          {
            "nll": 0.3349361717700958,
            "err": 0.1224999949336052,
            "missing_corr": 0.9228905173160256,
            "b": 0.7681577205657959,
            "pred_missing": 0.4607542157173157,
            "observed_missing": 0.3425000011920929
          },
          {
            "nll": 0.30076807737350464,
            "err": 0.11499999463558197,
            "missing_corr": 0.9735755481574243,
            "b": 0.7658361196517944,
            "pred_missing": 0.4581686854362488,
            "observed_missing": 0.33250001072883606
          },
          {
            "nll": 0.34624627232551575,
            "err": 0.14000000059604645,
            "missing_corr": 0.8828189879427482,
            "b": 0.7663794159889221,
            "pred_missing": 0.4717622399330139,
            "observed_missing": 0.3725000023841858
          },
          {
            "nll": 0.3216817080974579,
            "err": 0.14499999582767487,
            "missing_corr": 0.9267386520612199,
            "b": 0.7663792967796326,
            "pred_missing": 0.46759405732154846,
            "observed_missing": 0.3425000011920929
          }
        ],
        "mean_nll": 0.30827105790376663
      },
      {
        "config": {
          "lr": 0.01,
          "wd": 0.0
        },
        "per_seed": [
          {
            "nll": 0.2529575228691101,
            "err": 0.09749999642372131,
            "missing_corr": 0.8599101025078111,
            "b": 0.7472167015075684,
            "pred_missing": 0.4164726138114929,
            "observed_missing": 0.3125
          },
          {
            "nll": 0.3246569037437439,
            "err": 0.08999999612569809,
            "missing_corr": 0.9487513902968302,
            "b": 0.745702862739563,
            "pred_missing": 0.41046252846717834,
            "observed_missing": 0.3425000011920929
          },
          {
            "nll": 0.25593283772468567,
            "err": 0.11249999701976776,
            "missing_corr": 0.9430598899116818,
            "b": 0.7479268908500671,
            "pred_missing": 0.4195019602775574,
            "observed_missing": 0.33500000834465027
          },
          {
            "nll": 0.2860279083251953,
            "err": 0.11499999463558197,
            "missing_corr": 0.9281673470315555,
            "b": 0.7177038192749023,
            "pred_missing": 0.4063892066478729,
            "observed_missing": 0.3174999952316284
          },
          {
            "nll": 0.3488270044326782,
            "err": 0.125,
            "missing_corr": 0.8866245947416767,
            "b": 0.7491956949234009,
            "pred_missing": 0.4097135066986084,
            "observed_missing": 0.3425000011920929
          },
          {
            "nll": 0.29746517539024353,
            "err": 0.11749999970197678,
            "missing_corr": 0.9490765968164979,
            "b": 0.7454859614372253,
            "pred_missing": 0.41396957635879517,
            "observed_missing": 0.33250001072883606
          },
          {
            "nll": 0.3761008381843567,
            "err": 0.1525000035762787,
            "missing_corr": 0.8037557854015652,
            "b": 0.7294673919677734,
            "pred_missing": 0.4210052490234375,
            "observed_missing": 0.3725000023841858
          },
          {
            "nll": 0.3177599310874939,
            "err": 0.14000000059604645,
            "missing_corr": 0.9021314944562635,
            "b": 0.7316409945487976,
            "pred_missing": 0.41623786091804504,
            "observed_missing": 0.3425000011920929
          }
        ],
        "mean_nll": 0.3074660152196884
      },
      {
        "config": {
          "lr": 0.03,
          "wd": 0.0
        },
        "per_seed": [
          {
            "nll": 0.2912931442260742,
            "err": 0.10249999910593033,
            "missing_corr": 0.8220934083194001,
            "b": 0.790917158126831,
            "pred_missing": 0.3713911473751068,
            "observed_missing": 0.3125
          },
          {
            "nll": 0.35071563720703125,
            "err": 0.08249999582767487,
            "missing_corr": 0.9079289341663003,
            "b": 0.8076502680778503,
            "pred_missing": 0.33232399821281433,
            "observed_missing": 0.3425000011920929
          },
          {
            "nll": 0.24430160224437714,
            "err": 0.10999999940395355,
            "missing_corr": 0.9119544553233667,
            "b": 0.8473079800605774,
            "pred_missing": 0.3684086501598358,
            "observed_missing": 0.33500000834465027
          },
          {
            "nll": 0.29751330614089966,
            "err": 0.11249999701976776,
            "missing_corr": 0.8516922311869071,
            "b": 0.7000229358673096,
            "pred_missing": 0.3121287524700165,
            "observed_missing": 0.3174999952316284
          },
          {
            "nll": 0.35454967617988586,
            "err": 0.12999999523162842,
            "missing_corr": 0.9089462931313597,
            "b": 0.8252794742584229,
            "pred_missing": 0.32259058952331543,
            "observed_missing": 0.3425000011920929
          },
          {
            "nll": 0.28305864334106445,
            "err": 0.11999999731779099,
            "missing_corr": 0.9016399678001666,
            "b": 0.8164690732955933,
            "pred_missing": 0.3561153709888458,
            "observed_missing": 0.33250001072883606
          },
          {
            "nll": 0.47652503848075867,
            "err": 0.17749999463558197,
            "missing_corr": 0.7801492316430328,
            "b": 0.7549465298652649,
            "pred_missing": 0.3328292965888977,
            "observed_missing": 0.3725000023841858
          },
          {
            "nll": 0.32324108481407166,
            "err": 0.13499999046325684,
            "missing_corr": 0.878917930163105,
            "b": 0.7771673798561096,
            "pred_missing": 0.3450394868850708,
            "observed_missing": 0.3425000011920929
          }
        ],
        "mean_nll": 0.32764976657927036
      }
    ]
  },
  "comparison": {
    "delta_mean": 0.001200985163450241,
    "idea_wins": 4,
    "n_pairs": 8,
    "per_seed_diffs": [
      -0.006365716457366943,
      0.0046363770961761475,
      -0.00877833366394043,
      0.01223328709602356,
      0.013913571834564209,
      0.018020689487457275,
      -0.021302610635757446,
      -0.0027493834495544434
    ],
    "p_value": 0.79055,
    "mde": 0.01120430419306732,
    "mde_rel_pct": 3.658368763488904,
    "verdict": "no measurable effect",
    "system_worked": false
  },
  "custom_track": {
    "name": "uncertainty_missingness_classification",
    "file": "missingness_bench.py",
    "domain": "label_missingness"
  },
  "math_check": {
    "a_grad": -0.0441457297454092,
    "a_expected": -0.0441457297454092,
    "b_grad": 0.034629446878288096,
    "b_expected": 0.03462944687828812,
    "entropy_grad_norm": 0.10446851953879388,
    "gradient_match": true
  },
  "protocol": {
    "baseline_grid": [
      {
        "lr": 0.003,
        "wd": 0.0
      },
      {
        "lr": 0.003,
        "wd": 0.0001
      },
      {
        "lr": 0.01,
        "wd": 0.0
      },
      {
        "lr": 0.01,
        "wd": 0.0001
      },
      {
        "lr": 0.03,
        "wd": 0.0
      },
      {
        "lr": 0.03,
        "wd": 0.0001
      }
    ],
    "idea_grid": [
      0.003,
      0.01,
      0.03
    ],
    "paired_deltas_nll": [
      -0.006365716457366943,
      0.0046363770961761475,
      -0.00877833366394043,
      0.01223328709602356,
      0.013913571834564209,
      0.018020689487457275,
      -0.021302610635757446,
      -0.0027493834495544434
    ],
    "permutation_p": 0.79055,
    "seeds": [
      0,
      1,
      2,
      3,
      4,
      5,
      6,
      7
    ]
  },
  "mechanism_signature": {
    "predicted": "positive uncertainty-to-missingness slope and high predicted/observed missingness correlation",
    "observed_b_mean": 0.7392925396561623,
    "observed_missing_corr_mean": 0.9026846501454853,
    "confirmed": true
  }
}