{
  "bench_version": 1,
  "track": "tabular",
  "model": "mlp_tiny",
  "metric_direction": "lower is better",
  "n_seeds": 8,
  "baseline": {
    "best_cfg": {
      "lr": 0.006,
      "weight_decay": 0.0
    },
    "sweep": [
      {
        "cfg": {
          "lr": 0.0015,
          "weight_decay": 0.0
        },
        "mean": 20.46571445465088
      },
      {
        "cfg": {
          "lr": 0.003,
          "weight_decay": 0.0
        },
        "mean": 16.009180068969727
      },
      {
        "cfg": {
          "lr": 0.006,
          "weight_decay": 0.0
        },
        "mean": 13.129557490348816
      },
      {
        "cfg": {
          "lr": 0.003,
          "weight_decay": 0.0001
        },
        "mean": 15.930055141448975
      }
    ],
    "full": {
      "mean": 13.129557490348816,
      "std": 1.5250596544234398,
      "per_seed": [
        13.93191909790039,
        13.616814613342285,
        10.975268363952637,
        13.225879669189453,
        10.728046417236328,
        12.577923774719238,
        14.791933059692383,
        15.188674926757812
      ],
      "n": 8
    }
  },
  "idea": {
    "best_cfg": {
      "lr": 0.006,
      "weight_decay": 0.0
    },
    "sweep": [
      {
        "cfg": {
          "lr": 0.0015,
          "weight_decay": 0.0
        },
        "mean": 19.026683568954468,
        "std": 2.310262868739491,
        "per_seed": [
          20.679960250854492,
          15.556901931762695,
          18.93318748474121,
          17.145143508911133,
          19.445880889892578,
          16.939167022705078,
          20.18524742126465,
          23.327980041503906
        ],
        "signature_per_seed": [
          {
            "accepted": 35,
            "rejected": 25,
            "predicted_decrease_mean": 1.382528269290924,
            "observed_decrease_mean": 1.3716121673583985,
            "radius_final": 0.012100000000000001
          },
          {
            "accepted": 41,
            "rejected": 19,
            "predicted_decrease_mean": 2.0615332511866966,
            "observed_decrease_mean": 2.022140456408989,
            "radius_final": 0.014641000000000003
          },
          {
            "accepted": 40,
            "rejected": 20,
            "predicted_decrease_mean": 1.1156054258346557,
            "observed_decrease_mean": 1.1034204244613648,
            "radius_final": 0.011000000000000001
          },
          {
            "accepted": 41,
            "rejected": 19,
            "predicted_decrease_mean": 1.26738692674695,
            "observed_decrease_mean": 1.255918735411109,
            "radius_final": 0.011000000000000001
          },
          {
            "accepted": 38,
            "rejected": 22,
            "predicted_decrease_mean": 1.2735088334271782,
            "observed_decrease_mean": 1.2580657758210834,
            "radius_final": 0.01
          },
          {
            "accepted": 40,
            "rejected": 20,
            "predicted_decrease_mean": 1.3169351086020469,
            "observed_decrease_mean": 1.3040775299072265,
            "radius_final": 0.01
          },
          {
            "accepted": 41,
            "rejected": 19,
            "predicted_decrease_mean": 1.2663456913901538,
            "observed_decrease_mean": 1.2509966594416921,
            "radius_final": 0.013310000000000002
          },
          {
            "accepted": 45,
            "rejected": 15,
            "predicted_decrease_mean": 1.2946510824892257,
            "observed_decrease_mean": 1.281169213189019,
            "radius_final": 0.01806992220000001
          }
        ]
      },
      {
        "cfg": {
          "lr": 0.003,
          "weight_decay": 0.0
        },
        "mean": 15.578829765319824,
        "std": 1.2037474931922048,
        "per_seed": [
          16.088903427124023,
          13.419285774230957,
          15.963817596435547,
          14.6215238571167,
          14.568916320800781,
          15.911458969116211,
          17.396286010742188,
          16.660446166992188
        ],
        "signature_per_seed": [
          {
            "accepted": 32,
            "rejected": 28,
            "predicted_decrease_mean": 1.2606446049176157,
            "observed_decrease_mean": 1.1952399015426636,
            "radius_final": 0.0102
          },
          {
            "accepted": 33,
            "rejected": 27,
            "predicted_decrease_mean": 1.7287133065136997,
            "observed_decrease_mean": 1.6648587313565342,
            "radius_final": 0.012342000000000002
          },
          {
            "accepted": 38,
            "rejected": 22,
            "predicted_decrease_mean": 1.2891257283718962,
            "observed_decrease_mean": 1.2490642447220652,
            "radius_final": 0.011000000000000001
          },
          {
            "accepted": 45,
            "rejected": 15,
            "predicted_decrease_mean": 1.1980243287152714,
            "observed_decrease_mean": 1.1577167299058702,
            "radius_final": 0.011000000000000001
          },
          {
            "accepted": 35,
            "rejected": 25,
            "predicted_decrease_mean": 1.508543290410723,
            "observed_decrease_mean": 1.4751929146902902,
            "radius_final": 0.01
          },
          {
            "accepted": 35,
            "rejected": 25,
            "predicted_decrease_mean": 1.1945056153195246,
            "observed_decrease_mean": 1.1530200685773577,
            "radius_final": 0.01
          },
          {
            "accepted": 36,
            "rejected": 24,
            "predicted_decrease_mean": 1.228395100062092,
            "observed_decrease_mean": 1.1819042099846735,
            "radius_final": 0.01
          },
          {
            "accepted": 43,
            "rejected": 17,
            "predicted_decrease_mean": 1.962806150772024,
            "observed_decrease_mean": 1.9334695505541424,
            "radius_final": 0.014933820000000006
          }
        ]
      },
      {
        "cfg": {
          "lr": 0.006,
          "weight_decay": 0.0
        },
        "mean": 12.88543701171875,
        "std": 1.4828934325502452,
        "per_seed": [
          13.835183143615723,
          11.309344291687012,
          12.560839653015137,
          12.500975608825684,
          10.138545989990234,
          13.836167335510254,
          14.998884201049805,
          13.903555870056152
        ],
        "signature_per_seed": [
          {
            "accepted": 28,
            "rejected": 32,
            "predicted_decrease_mean": 1.379367763974837,
            "observed_decrease_mean": 1.2420953001294817,
            "radius_final": 0.010404000000000002
          },
          {
            "accepted": 26,
            "rejected": 34,
            "predicted_decrease_mean": 1.344854632822367,
            "observed_decrease_mean": 1.24173248731173,
            "radius_final": 0.01
          },
          {
            "accepted": 33,
            "rejected": 27,
            "predicted_decrease_mean": 1.2900403211965705,
            "observed_decrease_mean": 1.2163373918244333,
            "radius_final": 0.011000000000000001
          },
          {
            "accepted": 36,
            "rejected": 24,
            "predicted_decrease_mean": 1.089351099398401,
            "observed_decrease_mean": 1.0336363050672743,
            "radius_final": 0.011000000000000001
          },
          {
            "accepted": 38,
            "rejected": 22,
            "predicted_decrease_mean": 2.0978008941502164,
            "observed_decrease_mean": 1.8838776789213483,
            "radius_final": 0.01
          },
          {
            "accepted": 30,
            "rejected": 30,
            "predicted_decrease_mean": 1.233123182753722,
            "observed_decrease_mean": 1.1296147982279459,
            "radius_final": 0.01
          },
          {
            "accepted": 33,
            "rejected": 27,
            "predicted_decrease_mean": 1.392147899577112,
            "observed_decrease_mean": 1.2645280722415808,
            "radius_final": 0.01
          },
          {
            "accepted": 39,
            "rejected": 21,
            "predicted_decrease_mean": 1.6642315091613011,
            "observed_decrease_mean": 1.6168656226916192,
            "radius_final": 0.011220000000000003
          }
        ]
      },
      {
        "cfg": {
          "lr": 0.003,
          "weight_decay": 0.0001
        },
        "mean": 15.578837275505066,
        "std": 1.203698466172168,
        "per_seed": [
          16.08891487121582,
          13.419283866882324,
          15.963841438293457,
          14.621552467346191,
          14.569123268127441,
          15.911477088928223,
          17.396265029907227,
          16.660240173339844
        ],
        "signature_per_seed": [
          {
            "accepted": 32,
            "rejected": 28,
            "predicted_decrease_mean": 1.2606316241435707,
            "observed_decrease_mean": 1.1952308416366577,
            "radius_final": 0.0102
          },
          {
            "accepted": 33,
            "rejected": 27,
            "predicted_decrease_mean": 1.7287147830833087,
            "observed_decrease_mean": 1.6648634708288945,
            "radius_final": 0.012342000000000002
          },
          {
            "accepted": 38,
            "rejected": 22,
            "predicted_decrease_mean": 1.2891209054934352,
            "observed_decrease_mean": 1.2490592253835577,
            "radius_final": 0.011000000000000001
          },
          {
            "accepted": 45,
            "rejected": 15,
            "predicted_decrease_mean": 1.1980234457386865,
            "observed_decrease_mean": 1.1577159881591796,
            "radius_final": 0.011000000000000001
          },
          {
            "accepted": 35,
            "rejected": 25,
            "predicted_decrease_mean": 1.5085362383297511,
            "observed_decrease_mean": 1.475187737601144,
            "radius_final": 0.01
          },
          {
            "accepted": 35,
            "rejected": 25,
            "predicted_decrease_mean": 1.1944936403206416,
            "observed_decrease_mean": 1.1530044828142438,
            "radius_final": 0.01
          },
          {
            "accepted": 36,
            "rejected": 24,
            "predicted_decrease_mean": 1.228386132667462,
            "observed_decrease_mean": 1.1818928718566895,
            "radius_final": 0.01
          },
          {
            "accepted": 43,
            "rejected": 17,
            "predicted_decrease_mean": 1.9628009385277712,
            "observed_decrease_mean": 1.933466112890909,
            "radius_final": 0.014933820000000006
          }
        ]
      }
    ],
    "mean": 12.88543701171875,
    "std": 1.4828934325502452,
    "per_seed": [
      13.835183143615723,
      11.309344291687012,
      12.560839653015137,
      12.500975608825684,
      10.138545989990234,
      13.836167335510254,
      14.998884201049805,
      13.903555870056152
    ],
    "n": 8,
    "full": {
      "mean": 12.88543701171875,
      "std": 1.4828934325502452,
      "per_seed": [
        13.835183143615723,
        11.309344291687012,
        12.560839653015137,
        12.500975608825684,
        10.138545989990234,
        13.836167335510254,
        14.998884201049805,
        13.903555870056152
      ],
      "n": 8
    }
  },
  "comparison": {
    "delta_mean": -0.24412047863006592,
    "idea_wins": 5,
    "n_pairs": 8,
    "per_seed_diffs": [
      -0.09673595428466797,
      -2.3074703216552734,
      1.5855712890625,
      -0.7249040603637695,
      -0.5895004272460938,
      1.2582435607910156,
      0.20695114135742188,
      -1.2851190567016602
    ],
    "p_value": 0.6102,
    "mde": 1.0727371153900689,
    "mde_rel_pct": 8.170398097411958,
    "verdict": "no significant win",
    "system_worked": false
  },
  "mechanism_signature": {
    "mechanism_signature": {
      "predicted_decrease_mean": 1.4363646628793159,
      "observed_decrease_mean": 1.3285859570519265,
      "observed_to_predicted_ratio": 0.9261847601497202,
      "accepted_steps_mean": 32.875,
      "rejected_steps_mean": 27.125,
      "confirmed": true
    },
    "protocol": {
      "paired_seeds": [
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7
      ],
      "epochs": 15,
      "train_samples": 400,
      "test_samples": 200,
      "grid": [
        {
          "lr": 0.0015,
          "weight_decay": 0.0
        },
        {
          "lr": 0.003,
          "weight_decay": 0.0
        },
        {
          "lr": 0.006,
          "weight_decay": 0.0
        },
        {
          "lr": 0.003,
          "weight_decay": 0.0001
        }
      ]
    },
    "runtime_sec": 35.358980655670166
  }
}