1{
  2  "bench_version": 1,
  3  "track": "vision",
  4  "model": "cnn_small",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "best_cfg": {
  9      "lr": 0.003,
 10      "wd": 0.0
 11    },
 12    "sweep": [
 13      {
 14        "cfg": {
 15          "lr": 0.001,
 16          "wd": 0.0
 17        },
 18        "mean": 0.7899999767541885
 19      },
 20      {
 21        "cfg": {
 22          "lr": 0.001,
 23          "wd": 0.0001
 24        },
 25        "mean": 0.7899999767541885
 26      },
 27      {
 28        "cfg": {
 29          "lr": 0.003,
 30          "wd": 0.0
 31        },
 32        "mean": 0.7824999839067459
 33      },
 34      {
 35        "cfg": {
 36          "lr": 0.003,
 37          "wd": 0.0001
 38        },
 39        "mean": 0.8112499862909317
 40      },
 41      {
 42        "cfg": {
 43          "lr": 0.01,
 44          "wd": 0.0
 45        },
 46        "mean": 0.8837499767541885
 47      },
 48      {
 49        "cfg": {
 50          "lr": 0.01,
 51          "wd": 0.0001
 52        },
 53        "mean": 0.8912499696016312
 54      }
 55    ],
 56    "full": {
 57      "mean": 0.7974999845027924,
 58      "std": 0.031622772360742395,
 59      "per_seed": [
 60        0.7799999713897705,
 61        0.7599999904632568,
 62        0.8399999737739563,
 63        0.75,
 64        0.8149999976158142,
 65        0.7899999618530273,
 66        0.8399999737739563,
 67        0.8050000071525574
 68      ],
 69      "n": 8
 70    }
 71  },
 72  "idea": {
 73    "mean": 0.778749980032444,
 74    "std": 0.04196351799020234,
 75    "per_seed": [
 76      0.7549999952316284,
 77      0.6850000023841858,
 78      0.7949999570846558,
 79      0.7799999713897705,
 80      0.8149999976158142,
 81      0.7699999809265137,
 82      0.7999999523162842,
 83      0.8299999833106995
 84    ],
 85    "n": 8
 86  },
 87  "comparison": {
 88    "delta_mean": -0.018750004470348358,
 89    "idea_wins": 5,
 90    "n_pairs": 8,
 91    "per_seed_diffs": [
 92      -0.02499997615814209,
 93      -0.07499998807907104,
 94      -0.04500001668930054,
 95      0.029999971389770508,
 96      0.0,
 97      -0.019999980926513672,
 98      -0.04000002145767212,
 99      0.02499997615814209
100    ],
101    "p_value": 0.20525,
102    "mde": 0.029961011266907856,
103    "mde_rel_pct": 3.7568666895444873,
104    "verdict": "no significant win",
105    "system_worked": false
106  },
107  "mechanism_signature": {
108    "best_cfg": {
109      "lr": 0.003,
110      "wd": 0.0,
111      "p": 0.5
112    },
113    "p": 0.5,
114    "q_leaf_observed": [
115      0.6304718255996704,
116      0.596589982509613,
117      0.7500549554824829,
118      0.7355444431304932,
119      0.4830438196659088,
120      0.7880200147628784,
121      0.60601806640625,
122      0.7255614995956421
123    ],
124    "predicted_zero": [
125      0.6304718255996704,
126      0.596589982509613,
127      0.7500549554824829,
128      0.7355444431304932,
129      0.4830438196659088,
130      0.7880200147628784,
131      0.60601806640625,
132      0.7255614995956421
133    ],
134    "observed_zero": [
135      0.8817968964576721,
136      0.6903906464576721,
137      0.816162109375,
138      0.8238997459411621,
139      0.8374186158180237,
140      0.7389615774154663,
141      0.8830631375312805,
142      0.8237109184265137
143    ],
144    "confirmed": false,
145    "mean_abs_prediction_error": 0.1597769893705845
146  },
147  "idea_sweep": [
148    {
149      "cfg": {
150        "lr": 0.003,
151        "wd": 0.0,
152        "p": 0.25
153      },
154      "mean": 0.7931249737739563,
155      "per_seed": [
156        0.7749999761581421,
157        0.7899999618530273,
158        0.8050000071525574,
159        0.7699999809265137,
160        0.7949999570846558,
161        0.8449999690055847,
162        0.7999999523162842,
163        0.7649999856948853
164      ]
165    },
166    {
167      "cfg": {
168        "lr": 0.003,
169        "wd": 0.0,
170        "p": 0.5
171      },
172      "mean": 0.778749980032444,
173      "per_seed": [
174        0.7549999952316284,
175        0.6850000023841858,
176        0.7949999570846558,
177        0.7799999713897705,
178        0.8149999976158142,
179        0.7699999809265137,
180        0.7999999523162842,
181        0.8299999833106995
182      ]
183    },
184    {
185      "cfg": {
186        "lr": 0.003,
187        "wd": 0.0,
188        "p": 0.75
189      },
190      "mean": 0.7874999865889549,
191      "per_seed": [
192        0.7749999761581421,
193        0.8700000047683716,
194        0.7699999809265137,
195        0.75,
196        0.8349999785423279,
197        0.7649999856948853,
198        0.8050000071525574,
199        0.7299999594688416
200      ]
201    }
202  ],
203  "structural_match": "vision CNN spatial hierarchical pooling",
204  "epochs": 8,
205  "n_train": 400,
206  "n_test": 200
207}