{
  "bench_version": 1,
  "track": "tabular",
  "model": "mlp_tiny",
  "metric_direction": "lower is better",
  "n_seeds": 8,
  "baseline": {
    "best_cfg": {
      "lr": 0.01,
      "depth": 8
    },
    "sweep": [
      {
        "cfg": {
          "lr": 0.001,
          "depth": 1
        },
        "mean": 217.19686126708984
      },
      {
        "cfg": {
          "lr": 0.001,
          "depth": 2
        },
        "mean": 179.02721405029297
      },
      {
        "cfg": {
          "lr": 0.001,
          "depth": 4
        },
        "mean": 25.813298225402832
      },
      {
        "cfg": {
          "lr": 0.001,
          "depth": 8
        },
        "mean": 15.061450004577637
      },
      {
        "cfg": {
          "lr": 0.003,
          "depth": 1
        },
        "mean": 142.7078514099121
      },
      {
        "cfg": {
          "lr": 0.003,
          "depth": 2
        },
        "mean": 26.81587314605713
      },
      {
        "cfg": {
          "lr": 0.003,
          "depth": 4
        },
        "mean": 13.492629766464233
      },
      {
        "cfg": {
          "lr": 0.003,
          "depth": 8
        },
        "mean": 9.146221399307251
      },
      {
        "cfg": {
          "lr": 0.01,
          "depth": 1
        },
        "mean": 35.42466497421265
      },
      {
        "cfg": {
          "lr": 0.01,
          "depth": 2
        },
        "mean": 16.24430751800537
      },
      {
        "cfg": {
          "lr": 0.01,
          "depth": 4
        },
        "mean": 8.620880007743835
      },
      {
        "cfg": {
          "lr": 0.01,
          "depth": 8
        },
        "mean": 7.093326091766357
      }
    ],
    "full": {
      "mean": 7.334557235240936,
      "std": 0.824177040857451,
      "per_seed": [
        6.754525661468506,
        5.7614359855651855,
        7.475412368774414,
        8.381930351257324,
        7.79284143447876,
        6.628401756286621,
        8.117091178894043,
        7.764819145202637
      ],
      "n": 8
    }
  },
  "idea": {
    "mean": 23.1199893951416,
    "std": 5.121285566952831,
    "per_seed": [
      15.503646850585938,
      19.21010398864746,
      18.322921752929688,
      20.51085662841797,
      24.305091857910156,
      29.11318016052246,
      29.386009216308594,
      28.608104705810547
    ],
    "n": 8,
    "selected_cfg": {
      "lr": 0.01
    }
  },
  "comparison": {
    "delta_mean": 15.785432159900665,
    "idea_wins": 0,
    "n_pairs": 8,
    "per_seed_diffs": [
      8.749121189117432,
      13.448668003082275,
      10.847509384155273,
      12.128926277160645,
      16.512250423431396,
      22.48477840423584,
      21.26891803741455,
      20.84328556060791
    ],
    "p_value": 0.0081,
    "mde": 4.399418320126617,
    "mde_rel_pct": 59.982057253413714,
    "verdict": "idea worse (significant)",
    "system_worked": false
  },
  "mechanism_signature": {
    "math_sanity": {
      "formula_check": [
        {
          "M": 1,
          "predicted": 0.17300000000000004,
          "observed": 0.17465833333333333,
          "abs_error": 0.0016583333333332895
        },
        {
          "M": 2,
          "predicted": 0.3160710000000001,
          "observed": 0.3166833333333333,
          "abs_error": 0.0006123333333332148
        },
        {
          "M": 4,
          "predicted": 0.5322411229590001,
          "observed": 0.5315916666666667,
          "abs_error": 0.0006494562923333724
        },
        {
          "M": 8,
          "predicted": 0.7812016329493428,
          "observed": 0.7828416666666667,
          "abs_error": 0.0016400337173239077
        },
        {
          "M": 16,
          "predicted": 0.9521272745759659,
          "observed": 0.9519,
          "abs_error": 0.00022727457596594114
        },
        {
          "M": 32,
          "predicted": 0.9977082021604751,
          "observed": 0.997875,
          "abs_error": 0.00016679783952489036
        }
      ],
      "max_abs_error": 0.0016583333333332895,
      "stale_curve_optima": [
        {
          "stale": 0.0,
          "optimal_depth": 16
        },
        {
          "stale": 0.005,
          "optimal_depth": 7
        },
        {
          "stale": 0.015,
          "optimal_depth": 4
        },
        {
          "stale": 0.03,
          "optimal_depth": 3
        },
        {
          "stale": 0.05,
          "optimal_depth": 3
        }
      ]
    },
    "trained_model_signature": {
      "rho_hat": 0.13499999046325684,
      "predicted_escape": 0.44015932468572094,
      "observed_group_escape": 0.4399999976158142,
      "absolute_error": 0.00015932706990673395,
      "confirmed": true,
      "mean_depth": 1.0,
      "depth_trajectory": [
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1
      ]
    },
    "idea_lr_trials": [
      {
        "cfg": {
          "lr": 0.01
        },
        "result": {
          "mean": 23.1199893951416,
          "std": 5.121285566952831,
          "per_seed": [
            15.503646850585938,
            19.21010398864746,
            18.322921752929688,
            20.51085662841797,
            24.305091857910156,
            29.11318016052246,
            29.386009216308594,
            28.608104705810547
          ],
          "n": 8,
          "selected_cfg": {
            "lr": 0.01
          }
        }
      },
      {
        "cfg": {
          "lr": 0.001
        },
        "result": {
          "mean": 215.88136672973633,
          "std": 10.926122378078592,
          "per_seed": [
            228.2001190185547,
            218.22589111328125,
            219.18821716308594,
            203.1732177734375,
            216.90838623046875,
            202.43487548828125,
            205.00531005859375,
            233.9149169921875
          ],
          "n": 8
        }
      },
      {
        "cfg": {
          "lr": 0.003
        },
        "result": {
          "mean": 140.52062129974365,
          "std": 14.768009273435721,
          "per_seed": [
            146.7877197265625,
            148.9251708984375,
            133.30699157714844,
            141.8115234375,
            114.57950592041016,
            140.2008819580078,
            129.85032653808594,
            168.70285034179688
          ],
          "n": 8
        }
      }
    ],
    "track_choice": "tabular: replay-memory training intervention; same MLP and regression task in both arms"
  }
}