{
  "bench_version": 1,
  "track": "tabular",
  "model": "mlp_tiny",
  "metric_direction": "lower is better",
  "n_seeds": 8,
  "baseline": {
    "sweep": {
      "best_cfg": {
        "lr": 0.01
      },
      "sweep": [
        {
          "cfg": {
            "lr": 0.001
          },
          "mean": 32.33934259414673
        },
        {
          "cfg": {
            "lr": 0.003
          },
          "mean": 16.37940740585327
        },
        {
          "cfg": {
            "lr": 0.01
          },
          "mean": 10.050109148025513
        }
      ],
      "full": {
        "mean": 10.512850999832153,
        "std": 1.6350143489482907,
        "per_seed": [
          8.558663368225098,
          10.783540725708008,
          9.733381271362305,
          11.12485122680664,
          8.562338829040527,
          12.776362419128418,
          9.492801666259766,
          13.070868492126465
        ],
        "n": 8
      }
    },
    "best_config": {
      "lr": 0.01
    },
    "full": {
      "per_seed": [
        8.558663368225098,
        10.783540725708008,
        9.733381271362305,
        11.12485122680664,
        8.562338829040527,
        12.776362419128418,
        9.492801666259766,
        13.070868492126465
      ],
      "aux": [
        {
          "upward_fraction": 0.5784375071525574,
          "mean_left_gap": 0.009560184553265572,
          "codes": 16
        },
        {
          "upward_fraction": 0.5663281083106995,
          "mean_left_gap": 0.03742143139243126,
          "codes": 16
        },
        {
          "upward_fraction": 0.5204687118530273,
          "mean_left_gap": 0.07099631428718567,
          "codes": 16
        },
        {
          "upward_fraction": 0.5039452910423279,
          "mean_left_gap": 0.08094017207622528,
          "codes": 16
        },
        {
          "upward_fraction": 0.5638671517372131,
          "mean_left_gap": 0.05500025674700737,
          "codes": 16
        },
        {
          "upward_fraction": 0.19499999284744263,
          "mean_left_gap": 0.03867589682340622,
          "codes": 16
        },
        {
          "upward_fraction": 0.3338281214237213,
          "mean_left_gap": 0.015087975189089775,
          "codes": 16
        },
        {
          "upward_fraction": 0.2247656136751175,
          "mean_left_gap": 0.09521723538637161,
          "codes": 16
        }
      ],
      "mean": 10.512850999832153
    }
  },
  "idea": {
    "config": {
      "lr": 0.01
    },
    "per_seed": [
      9.773167610168457,
      11.56466007232666,
      13.178303718566895,
      14.895204544067383,
      10.585610389709473,
      15.57695484161377,
      10.29378890991211,
      14.999134063720703
    ],
    "aux": [
      {
        "upward_fraction": 0.0,
        "mean_left_gap": 0.09891153126955032,
        "codes": 16
      },
      {
        "upward_fraction": 0.0,
        "mean_left_gap": 0.1319166123867035,
        "codes": 16
      },
      {
        "upward_fraction": 0.0,
        "mean_left_gap": 0.20265933871269226,
        "codes": 16
      },
      {
        "upward_fraction": 0.0,
        "mean_left_gap": 0.20551525056362152,
        "codes": 16
      },
      {
        "upward_fraction": 0.0,
        "mean_left_gap": 0.1258753538131714,
        "codes": 16
      },
      {
        "upward_fraction": 0.0,
        "mean_left_gap": 0.14724649488925934,
        "codes": 16
      },
      {
        "upward_fraction": 0.0,
        "mean_left_gap": 0.09176831692457199,
        "codes": 16
      },
      {
        "upward_fraction": 0.0,
        "mean_left_gap": 0.2208053320646286,
        "codes": 16
      }
    ],
    "mean": 12.608353018760681,
    "sweep": [
      {
        "lr": 0.003,
        "per_seed": [
          17.26456069946289,
          16.607093811035156,
          19.64568328857422,
          18.041486740112305,
          17.236543655395508,
          19.011493682861328,
          15.007841110229492,
          20.771881103515625
        ],
        "aux": [
          {
            "upward_fraction": 0.0,
            "mean_left_gap": 0.11044175922870636,
            "codes": 16
          },
          {
            "upward_fraction": 0.0,
            "mean_left_gap": 0.13866443932056427,
            "codes": 16
          },
          {
            "upward_fraction": 0.0,
            "mean_left_gap": 0.18446868658065796,
            "codes": 16
          },
          {
            "upward_fraction": 0.0,
            "mean_left_gap": 0.20145590603351593,
            "codes": 16
          },
          {
            "upward_fraction": 0.0,
            "mean_left_gap": 0.14068220555782318,
            "codes": 16
          },
          {
            "upward_fraction": 0.0,
            "mean_left_gap": 0.13724036514759064,
            "codes": 16
          },
          {
            "upward_fraction": 0.0,
            "mean_left_gap": 0.10448167473077774,
            "codes": 16
          },
          {
            "upward_fraction": 0.0,
            "mean_left_gap": 0.22862808406352997,
            "codes": 16
          }
        ],
        "mean": 17.948323011398315
      },
      {
        "lr": 0.001,
        "per_seed": [
          27.97640609741211,
          42.11697006225586,
          70.37665557861328,
          48.89053726196289,
          47.3976936340332,
          83.98968505859375,
          23.241455078125,
          54.805789947509766
        ],
        "aux": [
          {
            "upward_fraction": 0.0,
            "mean_left_gap": 0.09334667772054672,
            "codes": 16
          },
          {
            "upward_fraction": 0.0,
            "mean_left_gap": 0.1060170829296112,
            "codes": 16
          },
          {
            "upward_fraction": 0.0,
            "mean_left_gap": 0.10366649180650711,
            "codes": 16
          },
          {
            "upward_fraction": 0.0,
            "mean_left_gap": 0.13664911687374115,
            "codes": 16
          },
          {
            "upward_fraction": 0.0,
            "mean_left_gap": 0.09772422909736633,
            "codes": 16
          },
          {
            "upward_fraction": 0.0,
            "mean_left_gap": 0.07773704826831818,
            "codes": 16
          },
          {
            "upward_fraction": 0.0,
            "mean_left_gap": 0.09394310414791107,
            "codes": 16
          },
          {
            "upward_fraction": 0.0,
            "mean_left_gap": 0.15413281321525574,
            "codes": 16
          }
        ],
        "mean": 49.84939908981323
      },
      {
        "lr": 0.01,
        "per_seed": [
          9.773167610168457,
          11.56466007232666,
          13.178303718566895,
          14.895204544067383,
          10.585610389709473,
          15.57695484161377,
          10.29378890991211,
          14.999134063720703
        ],
        "aux": [
          {
            "upward_fraction": 0.0,
            "mean_left_gap": 0.09891153126955032,
            "codes": 16
          },
          {
            "upward_fraction": 0.0,
            "mean_left_gap": 0.1319166123867035,
            "codes": 16
          },
          {
            "upward_fraction": 0.0,
            "mean_left_gap": 0.20265933871269226,
            "codes": 16
          },
          {
            "upward_fraction": 0.0,
            "mean_left_gap": 0.20551525056362152,
            "codes": 16
          },
          {
            "upward_fraction": 0.0,
            "mean_left_gap": 0.1258753538131714,
            "codes": 16
          },
          {
            "upward_fraction": 0.0,
            "mean_left_gap": 0.14724649488925934,
            "codes": 16
          },
          {
            "upward_fraction": 0.0,
            "mean_left_gap": 0.09176831692457199,
            "codes": 16
          },
          {
            "upward_fraction": 0.0,
            "mean_left_gap": 0.2208053320646286,
            "codes": 16
          }
        ],
        "mean": 12.608353018760681
      }
    ]
  },
  "comparison": {
    "delta_mean": 2.095502018928528,
    "idea_wins": 0,
    "n_pairs": 8,
    "per_seed_diffs": [
      1.2145042419433594,
      0.7811193466186523,
      3.44492244720459,
      3.770353317260742,
      2.0232715606689453,
      2.8005924224853516,
      0.8009872436523438,
      1.9282655715942383
    ],
    "p_value": 0.0081,
    "mde": 0.9656413775288165,
    "mde_rel_pct": 9.185342563537082,
    "verdict": "idea worse (significant)",
    "system_worked": false
  },
  "mechanism_signature": {
    "predicted": {
      "upward_fraction": 0.0,
      "left_gap_nonnegative": true
    },
    "observed": {
      "mean_upward_fraction": 0.0,
      "mean_left_gap": 0.15308727882802486
    },
    "confirmed": true
  }
}