1{
  2  "bench_version": 1,
  3  "track": "custom_tracks/rough_energy_regression",
  4  "model": "mlp_tiny",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "best_cfg": {
  9      "lr": 0.009,
 10      "weight_decay": 0.0001
 11    },
 12    "sweep": [
 13      {
 14        "cfg": {
 15          "lr": 0.001,
 16          "weight_decay": 0.0
 17        },
 18        "mean": 0.17962444573640823
 19      },
 20      {
 21        "cfg": {
 22          "lr": 0.001,
 23          "weight_decay": 0.0001
 24        },
 25        "mean": 0.17939040809869766
 26      },
 27      {
 28        "cfg": {
 29          "lr": 0.003,
 30          "weight_decay": 0.0
 31        },
 32        "mean": 0.13639282248914242
 33      },
 34      {
 35        "cfg": {
 36          "lr": 0.003,
 37          "weight_decay": 0.0001
 38        },
 39        "mean": 0.13660208880901337
 40      },
 41      {
 42        "cfg": {
 43          "lr": 0.009,
 44          "weight_decay": 0.0
 45        },
 46        "mean": 0.12427650578320026
 47      },
 48      {
 49        "cfg": {
 50          "lr": 0.009,
 51          "weight_decay": 0.0001
 52        },
 53        "mean": 0.12187139689922333
 54      }
 55    ],
 56    "full": {
 57      "mean": 0.11977773066610098,
 58      "std": 0.010381994134392314,
 59      "per_seed": [
 60        0.11285003274679184,
 61        0.12270589917898178,
 62        0.1380033791065216,
 63        0.11392627656459808,
 64        0.12507213652133942,
 65        0.10099826753139496,
 66        0.12710516154766083,
 67        0.11756069213151932
 68      ],
 69      "n": 8
 70    }
 71  },
 72  "idea": {
 73    "mean": 0.777425967156887,
 74    "std": 0.1318602192538638,
 75    "per_seed": [
 76      0.9373617172241211,
 77      0.9086369276046753,
 78      0.8551380634307861,
 79      0.555958092212677,
 80      0.8983770608901978,
 81      0.70047527551651,
 82      0.715994656085968,
 83      0.6474659442901611
 84    ],
 85    "n": 8,
 86    "best_cfg": {
 87      "lr": 0.001,
 88      "weight_decay": 0.0
 89    },
 90    "sweep": [
 91      {
 92        "cfg": {
 93          "lr": 0.001,
 94          "weight_decay": 0.0
 95        },
 96        "result": {
 97          "mean": 0.777425967156887,
 98          "std": 0.1318602192538638,
 99          "per_seed": [
100            0.9373617172241211,
101            0.9086369276046753,
102            0.8551380634307861,
103            0.555958092212677,
104            0.8983770608901978,
105            0.70047527551651,
106            0.715994656085968,
107            0.6474659442901611
108          ],
109          "n": 8
110        }
111      },
112      {
113        "cfg": {
114          "lr": 0.001,
115          "weight_decay": 0.0001
116        },
117        "result": {
118          "mean": 0.7774515226483345,
119          "std": 0.13189202326535374,
120          "per_seed": [
121            0.9375675916671753,
122            0.9086875915527344,
123            0.8551148772239685,
124            0.5559515357017517,
125            0.8983452916145325,
126            0.7004724144935608,
127            0.7159877419471741,
128            0.6474851369857788
129          ],
130          "n": 8
131        }
132      },
133      {
134        "cfg": {
135          "lr": 0.003,
136          "weight_decay": 0.0
137        },
138        "result": {
139          "mean": 1.001797802746296,
140          "std": 0.14249134156458224,
141          "per_seed": [
142            1.1306720972061157,
143            1.1415878534317017,
144            1.0970056056976318,
145            0.7530442476272583,
146            1.1628235578536987,
147            0.9271580576896667,
148            0.9430878758430481,
149            0.8590031266212463
150          ],
151          "n": 8
152        }
153      },
154      {
155        "cfg": {
156          "lr": 0.003,
157          "weight_decay": 0.0001
158        },
159        "result": {
160          "mean": 1.002461701631546,
161          "std": 0.14312791455365667,
162          "per_seed": [
163            1.1407548189163208,
164            1.1356585025787354,
165            1.0970054864883423,
166            0.7530419826507568,
167            1.163993239402771,
168            0.9271581768989563,
169            0.9430776238441467,
170            0.8590037822723389
171          ],
172          "n": 8
173        }
174      },
175      {
176        "cfg": {
177          "lr": 0.009,
178          "weight_decay": 0.0
179        },
180        "result": {
181          "mean": 1.1230522990226746,
182          "std": 0.04363725894180169,
183          "per_seed": [
184            1.1245875358581543,
185            1.1070117950439453,
186            1.18475341796875,
187            1.0596675872802734,
188            1.1676586866378784,
189            1.1612745523452759,
190            1.0625452995300293,
191            1.1169195175170898
192          ],
193          "n": 8
194        }
195      },
196      {
197        "cfg": {
198          "lr": 0.009,
199          "weight_decay": 0.0001
200        },
201        "result": {
202          "mean": 1.1208522766828537,
203          "std": 0.040913691981246045,
204          "per_seed": [
205            1.1206907033920288,
206            1.1047009229660034,
207            1.1829783916473389,
208            1.0658692121505737,
209            1.1569230556488037,
210            1.1598163843154907,
211            1.06282639503479,
212            1.1130131483078003
213          ],
214          "n": 8
215        }
216      }
217    ]
218  },
219  "comparison": {
220    "delta_mean": 0.6576482364907861,
221    "idea_wins": 0,
222    "n_pairs": 8,
223    "per_seed_diffs": [
224      0.8245116844773293,
225      0.7859310284256935,
226      0.7171346843242645,
227      0.4420318156480789,
228      0.7733049243688583,
229      0.599477007985115,
230      0.5888894945383072,
231      0.5299052521586418
232    ],
233    "p_value": 0.0081,
234    "mde": 0.11488058979741761,
235    "mde_rel_pct": 95.9114763308257,
236    "verdict": "idea worse (significant)",
237    "system_worked": false
238  },
239  "mechanism_signature": {
240    "custom_track": {
241      "name": "rough_energy_regression",
242      "file": "rough_track.py",
243      "domain": "optimization"
244    },
245    "mechanism_signature": {
246      "quantity": "prediction-vs-observed NN output roughness",
247      "predicted_bound": 10.0,
248      "observed_idea": 0.019673509523272514,
249      "observed_baseline_target_roughness": 0.7095075249671936,
250      "confirmed": true
251    }
252  }
253}