1{
  2  "bench_version": 1,
  3  "track": "vision",
  4  "model": "cnn_small",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "best_cfg": {
  9      "lr": 0.001
 10    },
 11    "sweep": [
 12      {
 13        "cfg": {
 14          "lr": 0.001
 15        },
 16        "mean": 0.88
 17      },
 18      {
 19        "cfg": {
 20          "lr": 0.003
 21        },
 22        "mean": 0.88625
 23      },
 24      {
 25        "cfg": {
 26          "lr": 0.01
 27        },
 28        "mean": 0.895
 29      }
 30    ],
 31    "full": {
 32      "mean": 0.88375,
 33      "std": 0.025829972899714785,
 34      "per_seed": [
 35        0.9075,
 36        0.885,
 37        0.9075,
 38        0.82,
 39        0.8875,
 40        0.89,
 41        0.8925,
 42        0.88
 43      ],
 44      "n": 8
 45    },
 46    "idea_grid": [
 47      {
 48        "lr": 0.001,
 49        "result": {
 50          "mean": 0.8799999999999999,
 51          "std": 0.0347311099736245,
 52          "per_seed": [
 53            0.9075,
 54            0.8825,
 55            0.9075,
 56            0.8225
 57          ],
 58          "n": 4
 59        }
 60      },
 61      {
 62        "lr": 0.003,
 63        "result": {
 64          "mean": 0.881875,
 65          "std": 0.04258722666481113,
 66          "per_seed": [
 67            0.9,
 68            0.9275,
 69            0.8875,
 70            0.8125
 71          ],
 72          "n": 4
 73        }
 74      },
 75      {
 76        "lr": 0.01,
 77        "result": {
 78          "mean": 0.895,
 79          "std": 0.012119199643540817,
 80          "per_seed": [
 81            0.9,
 82            0.9075,
 83            0.875,
 84            0.8975
 85          ],
 86          "n": 4
 87        }
 88      }
 89    ]
 90  },
 91  "idea": {
 92    "mean": 0.8843749999999999,
 93    "std": 0.02533371024938904,
 94    "per_seed": [
 95      0.9075,
 96      0.8825,
 97      0.9075,
 98      0.8225,
 99      0.8875,
100      0.89,
101      0.8975,
102      0.88
103    ],
104    "n": 8
105  },
106  "comparison": {
107    "delta_mean": 0.0006250000000000006,
108    "idea_wins": 1,
109    "n_pairs": 8,
110    "per_seed_diffs": [
111      0.0,
112      -0.0025000000000000577,
113      0.0,
114      0.0025000000000000577,
115      0.0,
116      0.0,
117      0.0050000000000000044,
118      0.0
119    ],
120    "p_value": 0.7525,
121    "mde": 0.0018529277478734423,
122    "mde_rel_pct": 0.20966650612429333,
123    "verdict": "no measurable effect",
124    "system_worked": false
125  },
126  "mechanism_signature": {
127    "prediction": "fused mean e-value equals uncertainty-weighted mean local e-values",
128    "observed": [
129      {
130        "accuracy": 0.0925,
131        "coverage": 1.0,
132        "avg_set_size": 10.0,
133        "mean_true_e": 0.9467349927047926,
134        "predicted_mean_true_e": 0.9467348347498856,
135        "convexity_abs_error": 1.5795490693193415e-07
136      },
137      {
138        "accuracy": 0.1175,
139        "coverage": 1.0,
140        "avg_set_size": 10.0,
141        "mean_true_e": 1.0163047070950022,
142        "predicted_mean_true_e": 1.0163049215843074,
143        "convexity_abs_error": 2.1448930520584497e-07
144      },
145      {
146        "accuracy": 0.0925,
147        "coverage": 1.0,
148        "avg_set_size": 10.0,
149        "mean_true_e": 1.047165460267897,
150        "predicted_mean_true_e": 1.0471653046459115,
151        "convexity_abs_error": 1.556219855203267e-07
152      },
153      {
154        "accuracy": 0.1775,
155        "coverage": 1.0,
156        "avg_set_size": 10.0,
157        "mean_true_e": 0.9733103156789212,
158        "predicted_mean_true_e": 0.973310362700516,
159        "convexity_abs_error": 4.702159472902423e-08
160      },
161      {
162        "accuracy": 0.1125,
163        "coverage": 1.0,
164        "avg_set_size": 10.0,
165        "mean_true_e": 0.9805528636845341,
166        "predicted_mean_true_e": 0.9805527689498554,
167        "convexity_abs_error": 9.473467865728225e-08
168      },
169      {
170        "accuracy": 0.11,
171        "coverage": 1.0,
172        "avg_set_size": 10.0,
173        "mean_true_e": 0.976847474581956,
174        "predicted_mean_true_e": 0.9768475761609023,
175        "convexity_abs_error": 1.0157894636542153e-07
176      },
177      {
178        "accuracy": 0.1025,
179        "coverage": 1.0,
180        "avg_set_size": 10.0,
181        "mean_true_e": 1.0040690733188393,
182        "predicted_mean_true_e": 1.0040692552907968,
183        "convexity_abs_error": 1.819719575024692e-07
184      },
185      {
186        "accuracy": 0.12,
187        "coverage": 1.0,
188        "avg_set_size": 10.0,
189        "mean_true_e": 0.9969966644785999,
190        "predicted_mean_true_e": 0.9969968797114046,
191        "convexity_abs_error": 2.1523280469004646e-07
192      }
193    ],
194    "confirmed": true,
195    "max_abs_error": 2.1523280469004646e-07
196  },
197  "idea_sweep": [
198    {
199      "lr": 0.001,
200      "result": {
201        "mean": 0.8799999999999999,
202        "std": 0.0347311099736245,
203        "per_seed": [
204          0.9075,
205          0.8825,
206          0.9075,
207          0.8225
208        ],
209        "n": 4
210      }
211    },
212    {
213      "lr": 0.003,
214      "result": {
215        "mean": 0.881875,
216        "std": 0.04258722666481113,
217        "per_seed": [
218          0.9,
219          0.9275,
220          0.8875,
221          0.8125
222        ],
223        "n": 4
224      }
225    },
226    {
227      "lr": 0.01,
228      "result": {
229        "mean": 0.895,
230        "std": 0.012119199643540817,
231        "per_seed": [
232          0.9,
233          0.9075,
234          0.875,
235          0.8975
236        ],
237        "n": 4
238      }
239    }
240  ],
241  "notes": "Vision selected because labels/candidate-label e-values are classification-specific; N_TRAIN/N_TEST=400 and 3 epochs were used for budget."
242}