1{
  2  "bench_version": 1,
  3  "track": "sequence",
  4  "model": "transformer_tiny",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "best_cfg": {
  9      "lr": 0.001,
 10      "temperature": 1.0
 11    },
 12    "sweep": [
 13      {
 14        "cfg": {
 15          "lr": 0.001,
 16          "temperature": 0.7
 17        },
 18        "mean": 0.6538640260696411
 19      },
 20      {
 21        "cfg": {
 22          "lr": 0.001,
 23          "temperature": 1.0
 24        },
 25        "mean": 0.6523745656013489
 26      },
 27      {
 28        "cfg": {
 29          "lr": 0.003,
 30          "temperature": 0.7
 31        },
 32        "mean": 0.6653543412685394
 33      },
 34      {
 35        "cfg": {
 36          "lr": 0.003,
 37          "temperature": 1.0
 38        },
 39        "mean": 0.6606301814317703
 40      },
 41      {
 42        "cfg": {
 43          "lr": 0.006,
 44          "temperature": 0.7
 45        },
 46        "mean": 0.7325459271669388
 47      },
 48      {
 49        "cfg": {
 50          "lr": 0.006,
 51          "temperature": 1.0
 52        },
 53        "mean": 0.7683279514312744
 54      }
 55    ],
 56    "full": {
 57      "mean": 0.6796555295586586,
 58      "std": 0.05923041132805598,
 59      "per_seed": [
 60        0.7301222085952759,
 61        0.6001890897750854,
 62        0.6041663885116577,
 63        0.6750205755233765,
 64        0.7771981358528137,
 65        0.6423909664154053,
 66        0.7304211258888245,
 67        0.6777357459068298
 68      ],
 69      "n": 8
 70    }
 71  },
 72  "idea": {
 73    "mean": 0.647096149623394,
 74    "std": 0.07594962062926218,
 75    "per_seed": [
 76      0.774990975856781,
 77      0.5947428941726685,
 78      0.5252495408058167,
 79      0.6463086605072021,
 80      0.7060304880142212,
 81      0.5773214101791382,
 82      0.7090907096862793,
 83      0.6430345177650452
 84    ],
 85    "n": 8
 86  },
 87  "comparison": {
 88    "delta_mean": -0.03255937993526459,
 89    "idea_wins": 7,
 90    "n_pairs": 8,
 91    "per_seed_diffs": [
 92      0.04486876726150513,
 93      -0.005446195602416992,
 94      -0.07891684770584106,
 95      -0.028711915016174316,
 96      -0.07116764783859253,
 97      -0.06506955623626709,
 98      -0.021330416202545166,
 99      -0.03470122814178467
100    ],
101    "p_value": 0.0673,
102    "mde": 0.033997422326021196,
103    "mde_rel_pct": 5.002154892802501,
104    "verdict": "no significant win",
105    "system_worked": false
106  },
107  "mechanism_signature": {
108    "trained_model": true,
109    "M": 32,
110    "observed_rmse_by_M": [
111      {
112        "M": 8,
113        "rmse": 0.4599074721336365
114      },
115      {
116        "M": 16,
117        "rmse": 0.35632288455963135
118      },
119      {
120        "M": 32,
121        "rmse": 0.22849832475185394
122      }
123    ],
124    "observed_slope": -0.5045800246521701,
125    "predicted_slope": -0.5,
126    "signed_rmse": 0.22849832475185394,
127    "unsigned_rmse": 0.38780272006988525,
128    "confirmed": true
129  },
130  "idea_sweep": [
131    {
132      "cfg": {
133        "lr": 0.001,
134        "m": 32
135      },
136      "result": {
137        "mean": 0.7032602652907372,
138        "std": 0.11299721951432162,
139        "per_seed": [
140          0.7925687432289124,
141          0.6065788269042969,
142          0.5842217206954956,
143          0.6156045198440552,
144          0.8893585801124573,
145          0.5813699960708618,
146          0.74463951587677,
147          0.8117402195930481
148        ],
149        "n": 8
150      }
151    },
152    {
153      "cfg": {
154        "lr": 0.003,
155        "m": 32
156      },
157      "result": {
158        "mean": 0.647096149623394,
159        "std": 0.07594962062926218,
160        "per_seed": [
161          0.774990975856781,
162          0.5947428941726685,
163          0.5252495408058167,
164          0.6463086605072021,
165          0.7060304880142212,
166          0.5773214101791382,
167          0.7090907096862793,
168          0.6430345177650452
169        ],
170        "n": 8
171      }
172    },
173    {
174      "cfg": {
175        "lr": 0.006,
176        "m": 32
177      },
178      "result": {
179        "mean": 0.9036180377006531,
180        "std": 0.0674238212671651,
181        "per_seed": [
182          0.9230982661247253,
183          0.871337890625,
184          0.8432034254074097,
185          0.7719256281852722,
186          0.9592460989952087,
187          0.9292699694633484,
188          1.0012027025222778,
189          0.9296603202819824
190        ],
191        "n": 8
192      }
193    }
194  ],
195  "sanity_check": {
196    "rows": [
197      {
198        "M": 16,
199        "signed_rmse": 0.3650483780576059,
200        "positive_rmse": 0.6758467186361339
201      },
202      {
203        "M": 32,
204        "signed_rmse": 0.20704154016476478,
205        "positive_rmse": 0.6506880673538492
206      },
207      {
208        "M": 64,
209        "signed_rmse": 0.16207412164394386,
210        "positive_rmse": 0.6397375154354732
211      },
212      {
213        "M": 128,
214        "signed_rmse": 0.12449882232224384,
215        "positive_rmse": 0.6258620222659124
216      },
217      {
218        "M": 256,
219        "signed_rmse": 0.08409067839559449,
220        "positive_rmse": 0.6281025092775255
221      }
222    ],
223    "observed_slope": -0.49699279208004493,
224    "predicted_slope": -0.5,
225    "positive_predicted_floor": 0.626260339536197,
226    "passed": true
227  },
228  "protocol": {
229    "seeds": [
230      0,
231      1,
232      2,
233      3,
234      4,
235      5,
236      6,
237      7
238    ],
239    "n_train": 400,
240    "n_test": 200,
241    "epochs": 6,
242    "baseline_grid": [
243      {
244        "lr": 0.001,
245        "temperature": 0.7
246      },
247      {
248        "lr": 0.001,
249        "temperature": 1.0
250      },
251      {
252        "lr": 0.003,
253        "temperature": 0.7
254      },
255      {
256        "lr": 0.003,
257        "temperature": 1.0
258      },
259      {
260        "lr": 0.006,
261        "temperature": 0.7
262      },
263      {
264        "lr": 0.006,
265        "temperature": 1.0
266      }
267    ],
268    "idea_grid": [
269      {
270        "lr": 0.001,
271        "m": 32
272      },
273      {
274        "lr": 0.003,
275        "m": 32
276      },
277      {
278        "lr": 0.006,
279        "m": 32
280      }
281    ]
282  }
283}