1{
  2  "bench_version": 1,
  3  "track": "tabular",
  4  "model": "mlp_tiny",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "best_cfg": {
  9      "lr": 0.01
 10    },
 11    "sweep": [
 12      {
 13        "cfg": {
 14          "lr": 0.001
 15        },
 16        "mean": 18.315088629722595
 17      },
 18      {
 19        "cfg": {
 20          "lr": 0.003
 21        },
 22        "mean": 13.52312970161438
 23      },
 24      {
 25        "cfg": {
 26          "lr": 0.01
 27        },
 28        "mean": 8.00898426771164
 29      }
 30    ],
 31    "full": {
 32      "per_seed": [
 33        7.518021106719971,
 34        7.098003387451172,
 35        7.579931259155273,
 36        8.855631828308105,
 37        7.726285457611084,
 38        7.675883769989014,
 39        9.252959251403809,
 40        8.365158081054688
 41      ],
 42      "mean": 8.00898426771164
 43    },
 44    "diagnostics": [
 45      {
 46        "metric": 7.518021106719971,
 47        "test_rank": 1.0162361860275269,
 48        "minmax_sv_ratio": 0.0,
 49        "seed": 0
 50      },
 51      {
 52        "metric": 7.098003387451172,
 53        "test_rank": 1.0113472938537598,
 54        "minmax_sv_ratio": 0.0,
 55        "seed": 1
 56      },
 57      {
 58        "metric": 7.579931259155273,
 59        "test_rank": 1.016098141670227,
 60        "minmax_sv_ratio": 0.0,
 61        "seed": 2
 62      },
 63      {
 64        "metric": 8.855631828308105,
 65        "test_rank": 1.0101362466812134,
 66        "minmax_sv_ratio": 0.0,
 67        "seed": 3
 68      },
 69      {
 70        "metric": 7.726285457611084,
 71        "test_rank": 1.0102609395980835,
 72        "minmax_sv_ratio": 0.0,
 73        "seed": 4
 74      },
 75      {
 76        "metric": 7.675883769989014,
 77        "test_rank": 1.0237842798233032,
 78        "minmax_sv_ratio": 0.0,
 79        "seed": 5
 80      },
 81      {
 82        "metric": 9.252959251403809,
 83        "test_rank": 1.0102062225341797,
 84        "minmax_sv_ratio": 0.0,
 85        "seed": 6
 86      },
 87      {
 88        "metric": 8.365158081054688,
 89        "test_rank": 1.0161538124084473,
 90        "minmax_sv_ratio": 0.0,
 91        "seed": 7
 92      }
 93    ]
 94  },
 95  "idea": {
 96    "per_seed": [
 97      7.499464988708496,
 98      7.094777584075928,
 99      7.588492393493652,
100      8.855938911437988,
101      7.727814674377441,
102      7.682166576385498,
103      9.250493049621582,
104      8.36063003540039
105    ],
106    "mean": 8.007472276687622,
107    "diagnostics": [
108      {
109        "metric": 7.499464988708496,
110        "test_rank": 1.016357183456421,
111        "minmax_sv_ratio": 0.0,
112        "seed": 0
113      },
114      {
115        "metric": 7.094777584075928,
116        "test_rank": 1.0115842819213867,
117        "minmax_sv_ratio": 0.0,
118        "seed": 1
119      },
120      {
121        "metric": 7.588492393493652,
122        "test_rank": 1.016180396080017,
123        "minmax_sv_ratio": 0.0,
124        "seed": 2
125      },
126      {
127        "metric": 8.855938911437988,
128        "test_rank": 1.0104186534881592,
129        "minmax_sv_ratio": 0.0,
130        "seed": 3
131      },
132      {
133        "metric": 7.727814674377441,
134        "test_rank": 1.0103423595428467,
135        "minmax_sv_ratio": 0.0,
136        "seed": 4
137      },
138      {
139        "metric": 7.682166576385498,
140        "test_rank": 1.023997187614441,
141        "minmax_sv_ratio": 0.0,
142        "seed": 5
143      },
144      {
145        "metric": 9.250493049621582,
146        "test_rank": 1.0103188753128052,
147        "minmax_sv_ratio": 0.0,
148        "seed": 6
149      },
150      {
151        "metric": 8.36063003540039,
152        "test_rank": 1.0162674188613892,
153        "minmax_sv_ratio": 0.0,
154        "seed": 7
155      }
156    ]
157  },
158  "comparison": {
159    "delta_mean": -0.001511991024017334,
160    "idea_wins": 4,
161    "n_pairs": 8,
162    "per_seed_diffs": [
163      -0.01855611801147461,
164      -0.0032258033752441406,
165      0.008561134338378906,
166      0.0003070831298828125,
167      0.0015292167663574219,
168      0.006282806396484375,
169      -0.0024662017822265625,
170      -0.004528045654296875
171    ],
172    "p_value": 0.69225,
173    "mde": 0.006911241884516106,
174    "mde_rel_pct": 0.08629361294139257,
175    "verdict": "no measurable effect",
176    "system_worked": false
177  },
178  "idea_grid": [
179    {
180      "lr": 0.001,
181      "mean": 18.320167779922485
182    },
183    {
184      "lr": 0.003,
185      "mean": 13.5223867893219
186    },
187    {
188      "lr": 0.01,
189      "mean": 8.007472276687622
190    }
191  ],
192  "paired_deltas": [
193    -0.01855611801147461,
194    -0.0032258033752441406,
195    0.008561134338378906,
196    0.0003070831298828125,
197    0.0015292167663574219,
198    0.006282806396484375,
199    -0.0024662017822265625,
200    -0.004528045654296875
201  ],
202  "paired_p_value": 0.69225,
203  "mechanism_signature": {
204    "prediction": "span regularization increases held-out effective rank",
205    "predicted_direction": "idea rank > baseline rank",
206    "baseline_mean_rank": 1.0142778903245926,
207    "idea_mean_rank": 1.0144332945346832,
208    "baseline_mean_sv_ratio": 0.0,
209    "idea_mean_sv_ratio": 0.0,
210    "confirmed": true
211  },
212  "elapsed_sec": 31.53953456878662
213}