1{
  2  "bench_version": 1,
  3  "track": "dynamics",
  4  "model": "rnn_small",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "best_cfg": {
  9      "lr": 0.005,
 10      "epochs": 20,
 11      "batch": 128
 12    },
 13    "sweep": [
 14      {
 15        "cfg": {
 16          "lr": 0.001,
 17          "epochs": 20,
 18          "batch": 128
 19        },
 20        "mean": 0.022121376357972622
 21      },
 22      {
 23        "cfg": {
 24          "lr": 0.002,
 25          "epochs": 20,
 26          "batch": 128
 27        },
 28        "mean": 0.016339546535164118
 29      },
 30      {
 31        "cfg": {
 32          "lr": 0.003,
 33          "epochs": 20,
 34          "batch": 128
 35        },
 36        "mean": 0.012747961794957519
 37      },
 38      {
 39        "cfg": {
 40          "lr": 0.005,
 41          "epochs": 20,
 42          "batch": 128
 43        },
 44        "mean": 0.009133224608376622
 45      }
 46    ],
 47    "full": {
 48      "mean": 0.008694025862496346,
 49      "std": 0.0014232314536370147,
 50      "per_seed": [
 51        0.007357772905379534,
 52        0.010028759948909283,
 53        0.007755488622933626,
 54        0.011390876956284046,
 55        0.006608752999454737,
 56        0.008554458618164062,
 57        0.008871559053659439,
 58        0.008984537795186043
 59      ],
 60      "n": 8
 61    }
 62  },
 63  "idea": {
 64    "mean": 0.004228192905429751,
 65    "std": 0.000991539149802616,
 66    "per_seed": [
 67      0.003527658525854349,
 68      0.0032258331775665283,
 69      0.0044794389978051186,
 70      0.00498495576903224,
 71      0.0034572044387459755,
 72      0.003025196958333254,
 73      0.005306838545948267,
 74      0.005818416830152273
 75    ],
 76    "n": 8,
 77    "config": {
 78      "lr": 0.005,
 79      "epochs": 20,
 80      "batch": 128
 81    },
 82    "records": [
 83      {
 84        "seed": 0,
 85        "lr": 0.005,
 86        "metric": 0.003527658525854349,
 87        "last_loss": 0.005181310921907425,
 88        "inverse_error": 1.1013007679139264e-05,
 89        "condition_mean": 1.57774817943573,
 90        "observed_latent_mutation_mean": 1.7316803932189941,
 91        "predicted_vs_observed_probe": {
 92          "predicted_inverse_error": 0.0,
 93          "observed_inverse_error": 1.1013007679139264e-05,
 94          "observed_exchange_mean": 5.360846996307373
 95        }
 96      },
 97      {
 98        "seed": 1,
 99        "lr": 0.005,
100        "metric": 0.0032258331775665283,
101        "last_loss": 0.002949269898235798,
102        "inverse_error": 7.703845767537132e-06,
103        "condition_mean": 4.959653377532959,
104        "observed_latent_mutation_mean": 1.1563115119934082,
105        "predicted_vs_observed_probe": {
106          "predicted_inverse_error": 0.0,
107          "observed_inverse_error": 7.703845767537132e-06,
108          "observed_exchange_mean": 2.966520309448242
109        }
110      },
111      {
112        "seed": 2,
113        "lr": 0.005,
114        "metric": 0.0044794389978051186,
115        "last_loss": 0.004607265554368496,
116        "inverse_error": 1.0601612302707508e-05,
117        "condition_mean": 8.03824234008789,
118        "observed_latent_mutation_mean": 1.345355749130249,
119        "predicted_vs_observed_probe": {
120          "predicted_inverse_error": 0.0,
121          "observed_inverse_error": 1.0601612302707508e-05,
122          "observed_exchange_mean": 4.600798606872559
123        }
124      },
125      {
126        "seed": 3,
127        "lr": 0.005,
128        "metric": 0.00498495576903224,
129        "last_loss": 0.0036369254160672428,
130        "inverse_error": 7.407511475321371e-06,
131        "condition_mean": 4.844733238220215,
132        "observed_latent_mutation_mean": 1.294931173324585,
133        "predicted_vs_observed_probe": {
134          "predicted_inverse_error": 0.0,
135          "observed_inverse_error": 7.407511475321371e-06,
136          "observed_exchange_mean": 3.5481743812561035
137        }
138      },
139      {
140        "seed": 4,
141        "lr": 0.005,
142        "metric": 0.0034572044387459755,
143        "last_loss": 0.0035951392445713283,
144        "inverse_error": 8.508128303219564e-06,
145        "condition_mean": 5.653021335601807,
146        "observed_latent_mutation_mean": 3.2120485305786133,
147        "predicted_vs_observed_probe": {
148          "predicted_inverse_error": 0.0,
149          "observed_inverse_error": 8.508128303219564e-06,
150          "observed_exchange_mean": 17.611797332763672
151        }
152      },
153      {
154        "seed": 5,
155        "lr": 0.005,
156        "metric": 0.003025196958333254,
157        "last_loss": 0.004070228198543191,
158        "inverse_error": 9.308403605245985e-06,
159        "condition_mean": 7.194035053253174,
160        "observed_latent_mutation_mean": 0.8057935237884521,
161        "predicted_vs_observed_probe": {
162          "predicted_inverse_error": 0.0,
163          "observed_inverse_error": 9.308403605245985e-06,
164          "observed_exchange_mean": 1.7656463384628296
165        }
166      },
167      {
168        "seed": 6,
169        "lr": 0.005,
170        "metric": 0.005306838545948267,
171        "last_loss": 0.0037229776568710802,
172        "inverse_error": 6.436449439206626e-06,
173        "condition_mean": 3.7536144256591797,
174        "observed_latent_mutation_mean": 1.0492488145828247,
175        "predicted_vs_observed_probe": {
176          "predicted_inverse_error": 0.0,
177          "observed_inverse_error": 6.436449439206626e-06,
178          "observed_exchange_mean": 2.438058853149414
179        }
180      },
181      {
182        "seed": 7,
183        "lr": 0.005,
184        "metric": 0.005818416830152273,
185        "last_loss": 0.004587573418393731,
186        "inverse_error": 1.0803267286974005e-05,
187        "condition_mean": 6.939658164978027,
188        "observed_latent_mutation_mean": 0.627605140209198,
189        "predicted_vs_observed_probe": {
190          "predicted_inverse_error": 0.0,
191          "observed_inverse_error": 1.0803267286974005e-05,
192          "observed_exchange_mean": 1.0146958827972412
193        }
194      }
195    ],
196    "sweep": [
197      {
198        "cfg": {
199          "lr": 0.001,
200          "epochs": 20,
201          "batch": 128
202        },
203        "mean": 0.02207344933412969
204      },
205      {
206        "cfg": {
207          "lr": 0.002,
208          "epochs": 20,
209          "batch": 128
210        },
211        "mean": 0.011823555920273066
212      },
213      {
214        "cfg": {
215          "lr": 0.003,
216          "epochs": 20,
217          "batch": 128
218        },
219        "mean": 0.007646026671864092
220      },
221      {
222        "cfg": {
223          "lr": 0.005,
224          "epochs": 20,
225          "batch": 128
226        },
227        "mean": 0.004228192905429751
228      }
229    ]
230  },
231  "comparison": {
232    "delta_mean": -0.0044658329570665956,
233    "idea_wins": 8,
234    "n_pairs": 8,
235    "per_seed_diffs": [
236      -0.0038301143795251846,
237      -0.006802926771342754,
238      -0.0032760496251285076,
239      -0.006405921187251806,
240      -0.003151548560708761,
241      -0.005529261659830809,
242      -0.003564720507711172,
243      -0.0031661209650337696
244    ],
245    "p_value": 0.0081,
246    "mde": 0.0012800509368697734,
247    "mde_rel_pct": 14.723339418526043,
248    "verdict": "idea better (significant)",
249    "system_worked": true
250  },
251  "protocol_note": "8 paired seeds; baseline sweep and idea sweep share lr union; official train_model",
252  "mechanism_signature": {
253    "predicted_inverse_error": 0.0,
254    "observed_inverse_error_mean": 8.972778232418932e-06,
255    "observed_inverse_error_max": 1.1013007679139264e-05,
256    "observed_condition_mean": 5.370088264346123,
257    "confirmed": true
258  },
259  "custom_track": null
260}