1{
  2  "bench_version": 1,
  3  "track": "tabular",
  4  "model": "mlp_tiny",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "best_cfg": {
  9      "lr": 0.03
 10    },
 11    "sweep": [
 12      {
 13        "cfg": {
 14          "lr": 0.001
 15        },
 16        "mean": 24.142436981201172
 17      },
 18      {
 19        "cfg": {
 20          "lr": 0.003
 21        },
 22        "mean": 13.75401242574056
 23      },
 24      {
 25        "cfg": {
 26          "lr": 0.01
 27        },
 28        "mean": 8.325917879740397
 29      },
 30      {
 31        "cfg": {
 32          "lr": 0.03
 33        },
 34        "mean": 6.865411599477132
 35      }
 36    ],
 37    "full": {
 38      "mean": 7.418325066566467,
 39      "std": 0.9955428790176792,
 40      "per_seed": [
 41        6.716945648193359,
 42        5.94451379776001,
 43        7.934775352478027,
 44        9.200451850891113,
 45        7.416287899017334,
 46        6.373697280883789,
 47        8.270235061645508,
 48        7.489693641662598
 49      ],
 50      "n": 8
 51    }
 52  },
 53  "idea": {
 54    "mean": 124.02495861053467,
 55    "std": 19.888619557248344,
 56    "per_seed": [
 57      133.7542266845703,
 58      95.59589385986328,
 59      161.67819213867188,
 60      123.55552673339844,
 61      96.50812530517578,
 62      130.4013214111328,
 63      120.75763702392578,
 64      129.94874572753906
 65    ],
 66    "n": 8,
 67    "chosen_cfg": {
 68      "lr": 0.03
 69    },
 70    "trials": [
 71      {
 72        "cfg": {
 73          "lr": 0.003
 74        },
 75        "mean": 235.4395955403646
 76      },
 77      {
 78        "cfg": {
 79          "lr": 0.01
 80        },
 81        "mean": 223.33053588867188
 82      },
 83      {
 84        "cfg": {
 85          "lr": 0.03
 86        },
 87        "mean": 130.3427708943685
 88      }
 89    ]
 90  },
 91  "comparison": {
 92    "delta_mean": 116.6066335439682,
 93    "idea_wins": 0,
 94    "n_pairs": 8,
 95    "per_seed_diffs": [
 96      127.03728103637695,
 97      89.65138006210327,
 98      153.74341678619385,
 99      114.35507488250732,
100      89.09183740615845,
101      124.02762413024902,
102      112.48740196228027,
103      122.45905208587646
104    ],
105    "p_value": 0.0081,
106    "mde": 17.547701974609687,
107    "mde_rel_pct": 236.54533627402162,
108    "verdict": "idea worse (significant)",
109    "system_worked": false
110  },
111  "mechanism_signature": {
112    "predicted": {
113      "transport_error": 0.0,
114      "call_reduction_fraction": 0.5
115    },
116    "observed": {
117      "mean_transport_error": 1.59325485071804e-06,
118      "mean_idea_grad_calls": 73.0,
119      "mean_baseline_grad_calls": 72.0,
120      "call_reduction_fraction": -0.013888888888888888
121    },
122    "confirmed": false
123  }
124}