1{
  2  "bench_version": 1,
  3  "track": "tabular",
  4  "model": "mlp_tiny",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "best_cfg": {
  9      "lr": 0.012,
 10      "beta": 0.85
 11    },
 12    "sweep": [
 13      {
 14        "cfg": {
 15          "lr": 0.003,
 16          "beta": 0.85
 17        },
 18        "mean": 7.4412044286727905
 19      },
 20      {
 21        "cfg": {
 22          "lr": 0.003,
 23          "beta": 0.95
 24        },
 25        "mean": 7.356154918670654
 26      },
 27      {
 28        "cfg": {
 29          "lr": 0.006,
 30          "beta": 0.85
 31        },
 32        "mean": 7.447907209396362
 33      },
 34      {
 35        "cfg": {
 36          "lr": 0.006,
 37          "beta": 0.95
 38        },
 39        "mean": 6.9799922704696655
 40      },
 41      {
 42        "cfg": {
 43          "lr": 0.012,
 44          "beta": 0.85
 45        },
 46        "mean": 6.368864178657532
 47      },
 48      {
 49        "cfg": {
 50          "lr": 0.012,
 51          "beta": 0.95
 52        },
 53        "mean": 11.507428050041199
 54      }
 55    ],
 56    "full": {
 57      "mean": 6.102685570716858,
 58      "std": 1.0114630114444267,
 59      "per_seed": [
 60        5.218059539794922,
 61        7.1943206787109375,
 62        6.8049211502075195,
 63        6.258155345916748,
 64        4.909553527832031,
 65        4.723381042480469,
 66        7.64085054397583,
 67        6.072242736816406
 68      ],
 69      "n": 8,
 70      "details": [
 71        {
 72          "metric": 5.218059539794922,
 73          "grad_evals": 252
 74        },
 75        {
 76          "metric": 7.1943206787109375,
 77          "grad_evals": 252
 78        },
 79        {
 80          "metric": 6.8049211502075195,
 81          "grad_evals": 252
 82        },
 83        {
 84          "metric": 6.258155345916748,
 85          "grad_evals": 252
 86        },
 87        {
 88          "metric": 4.909553527832031,
 89          "grad_evals": 252
 90        },
 91        {
 92          "metric": 4.723381042480469,
 93          "grad_evals": 252
 94        },
 95        {
 96          "metric": 7.64085054397583,
 97          "grad_evals": 252
 98        },
 99        {
100          "metric": 6.072242736816406,
101          "grad_evals": 252
102        }
103      ]
104    }
105  },
106  "idea": {
107    "mean": 82.47423553466797,
108    "std": 62.85156905560213,
109    "per_seed": [
110      85.1873779296875,
111      61.3437385559082,
112      54.8747444152832,
113      16.73917007446289,
114      198.2187042236328,
115      37.13703918457031,
116      172.99974060058594,
117      33.29336929321289
118    ],
119    "n": 8,
120    "details": [
121      {
122        "metric": 85.1873779296875,
123        "grad_evals": 252,
124        "cert_rate_mean": 0.3926487951538228,
125        "cert_rate_n": 125
126      },
127      {
128        "metric": 61.3437385559082,
129        "grad_evals": 252,
130        "cert_rate_mean": 0.2918240217992533,
131        "cert_rate_n": 125
132      },
133      {
134        "metric": 54.8747444152832,
135        "grad_evals": 252,
136        "cert_rate_mean": -0.11212525637936967,
137        "cert_rate_n": 125
138      },
139      {
140        "metric": 16.73917007446289,
141        "grad_evals": 252,
142        "cert_rate_mean": 0.1972534224451282,
143        "cert_rate_n": 125
144      },
145      {
146        "metric": 198.2187042236328,
147        "grad_evals": 252,
148        "cert_rate_mean": 0.15344815565986625,
149        "cert_rate_n": 125
150      },
151      {
152        "metric": 37.13703918457031,
153        "grad_evals": 252,
154        "cert_rate_mean": 0.0256587405843517,
155        "cert_rate_n": 125
156      },
157      {
158        "metric": 172.99974060058594,
159        "grad_evals": 252,
160        "cert_rate_mean": 0.4365923490884733,
161        "cert_rate_n": 125
162      },
163      {
164        "metric": 33.29336929321289,
165        "grad_evals": 252,
166        "cert_rate_mean": 0.2754857404526831,
167        "cert_rate_n": 125
168      }
169    ]
170  },
171  "comparison": {
172    "delta_mean": 76.37154996395111,
173    "idea_wins": 0,
174    "n_pairs": 8,
175    "per_seed_diffs": [
176      79.96931838989258,
177      54.149417877197266,
178      48.069823265075684,
179      10.481014728546143,
180      193.30915069580078,
181      32.413658142089844,
182      165.3588900566101,
183      27.221126556396484
184    ],
185    "p_value": 0.0081,
186    "mde": 56.17144123095511,
187    "mde_rel_pct": 920.4380691099063,
188    "verdict": "idea worse (significant)",
189    "system_worked": false
190  },
191  "mechanism_signature": {
192    "idea_sweep": [
193      {
194        "cfg": {
195          "lr": 0.003,
196          "gamma": 0.1
197        },
198        "mean": 54.53625774383545
199      },
200      {
201        "cfg": {
202          "lr": 0.006,
203          "gamma": 0.1
204        },
205        "mean": 76.64579582214355
206      },
207      {
208        "cfg": {
209          "lr": 0.012,
210          "gamma": 0.1
211        },
212        "mean": 63.562326431274414
213      }
214    ],
215    "mechanism_signature": {
216      "prediction": "certificate conformal rate ~= gamma",
217      "predicted": 0.1,
218      "observed": 0.20759824610052613,
219      "absolute_error": 0.10759824610052612,
220      "n_models": 8,
221      "confirmed": false
222    },
223    "budget": {
224      "epochs": 18,
225      "batch": 64,
226      "baseline_gradient_evals": 252,
227      "idea_gradient_evals": 252
228    }
229  }
230}