1{
  2  "bench_version": 1,
  3  "track": "dynamics",
  4  "model": "rnn_small",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "best_cfg": {
  9      "lr": 0.01,
 10      "clip_margin": 0.0
 11    },
 12    "sweep": [
 13      {
 14        "cfg": {
 15          "lr": 0.001,
 16          "clip_margin": 0.0
 17        },
 18        "mean": 0.008678926387801766
 19      },
 20      {
 21        "cfg": {
 22          "lr": 0.001,
 23          "clip_margin": 0.03
 24        },
 25        "mean": 0.010034722508862615
 26      },
 27      {
 28        "cfg": {
 29          "lr": 0.001,
 30          "clip_margin": 0.08
 31        },
 32        "mean": 0.012918951804749668
 33      },
 34      {
 35        "cfg": {
 36          "lr": 0.003,
 37          "clip_margin": 0.0
 38        },
 39        "mean": 0.006641886895522475
 40      },
 41      {
 42        "cfg": {
 43          "lr": 0.003,
 44          "clip_margin": 0.03
 45        },
 46        "mean": 0.007589806220494211
 47      },
 48      {
 49        "cfg": {
 50          "lr": 0.003,
 51          "clip_margin": 0.08
 52        },
 53        "mean": 0.009543078020215034
 54      },
 55      {
 56        "cfg": {
 57          "lr": 0.01,
 58          "clip_margin": 0.0
 59        },
 60        "mean": 0.005315697111655027
 61      },
 62      {
 63        "cfg": {
 64          "lr": 0.01,
 65          "clip_margin": 0.03
 66        },
 67        "mean": 0.006531292689032853
 68      },
 69      {
 70        "cfg": {
 71          "lr": 0.01,
 72          "clip_margin": 0.08
 73        },
 74        "mean": 0.008570412872359157
 75      }
 76    ],
 77    "full": {
 78      "mean": 0.0046574123844038695,
 79      "std": 0.0013966285668353233,
 80      "per_seed": [
 81        0.003739116946235299,
 82        0.004545919597148895,
 83        0.005411764141172171,
 84        0.007565987762063742,
 85        0.004889188800007105,
 86        0.002318228594958782,
 87        0.00456235371530056,
 88        0.004226739518344402
 89      ],
 90      "n": 8
 91    }
 92  },
 93  "idea": {
 94    "mean": 0.0047074658796191216,
 95    "std": 0.0014044730096107532,
 96    "per_seed": [
 97      0.003767308546230197,
 98      0.004722419194877148,
 99      0.005457298830151558,
100      0.007620518561452627,
101      0.004916135687381029,
102      0.002345199463889003,
103      0.004579263739287853,
104      0.0042515830136835575
105    ],
106    "n": 8
107  },
108  "comparison": {
109    "delta_mean": 5.005349521525204e-05,
110    "idea_wins": 0,
111    "n_pairs": 8,
112    "per_seed_diffs": [
113      2.8191599994897842e-05,
114      0.0001764995977282524,
115      4.553468897938728e-05,
116      5.45307993888855e-05,
117      2.6946887373924255e-05,
118      2.6970868930220604e-05,
119      1.6910023987293243e-05,
120      2.4843495339155197e-05
121    ],
122    "p_value": 0.0081,
123    "mde": 4.3914032994549896e-05,
124    "mde_rel_pct": 0.9428847903098175,
125    "verdict": "idea worse (significant)",
126    "system_worked": false
127  },
128  "mechanism_signature": {
129    "selected_idea_cfg": {
130      "lr": 0.01,
131      "kappa": 0.5
132    },
133    "baseline_best_cfg": {
134      "lr": 0.01,
135      "clip_margin": 0.0
136    },
137    "idea_sweep": [
138      {
139        "cfg": {
140          "lr": 0.001,
141          "kappa": 0.5
142        },
143        "mean_4seed": 0.008722120779566467
144      },
145      {
146        "cfg": {
147          "lr": 0.001,
148          "kappa": 1.0
149        },
150        "mean_4seed": 0.008760003955103457
151      },
152      {
153        "cfg": {
154          "lr": 0.001,
155          "kappa": 2.0
156        },
157        "mean_4seed": 0.008845672477036715
158      },
159      {
160        "cfg": {
161          "lr": 0.003,
162          "kappa": 0.5
163        },
164        "mean_4seed": 0.006680505815893412
165      },
166      {
167        "cfg": {
168          "lr": 0.003,
169          "kappa": 1.0
170        },
171        "mean_4seed": 0.006711311521939933
172      },
173      {
174        "cfg": {
175          "lr": 0.003,
176          "kappa": 2.0
177        },
178        "mean_4seed": 0.0067982005421072245
179      },
180      {
181        "cfg": {
182          "lr": 0.01,
183          "kappa": 0.5
184        },
185        "mean_4seed": 0.005391886283177882
186      },
187      {
188        "cfg": {
189          "lr": 0.01,
190          "kappa": 1.0
191        },
192        "mean_4seed": 0.005438438267447054
193      },
194      {
195        "cfg": {
196          "lr": 0.01,
197          "kappa": 2.0
198        },
199        "mean_4seed": 0.005516079894732684
200      }
201    ],
202    "mechanism_signature": {
203      "prediction": "adaptive ebar tracks observed model residual and larger kappa increases intervention",
204      "predicted_vs_observed_mae": 1.7471262253820896e-05,
205      "relative_mae": 0.007133921903418612,
206      "rows": [
207        {
208          "seed": 0,
209          "predicted_ebar": 0.002444567857310176,
210          "observed_residual": 0.0024249006528407335,
211          "intervention": 0.007492117118090391,
212          "test_mse": 0.003767308546230197
213        },
214        {
215          "seed": 1,
216          "predicted_ebar": 0.002442610915750265,
217          "observed_residual": 0.00243778177537024,
218          "intervention": 0.0064279804937541485,
219          "test_mse": 0.004722419194877148
220        },
221        {
222          "seed": 2,
223          "predicted_ebar": 0.0024595605209469795,
224          "observed_residual": 0.0024749869480729103,
225          "intervention": 0.014962565153837204,
226          "test_mse": 0.005457298830151558
227        },
228        {
229          "seed": 3,
230          "predicted_ebar": 0.0024496361147612333,
231          "observed_residual": 0.002463151467964053,
232          "intervention": 0.015951024368405342,
233          "test_mse": 0.007620518561452627
234        },
235        {
236          "seed": 4,
237          "predicted_ebar": 0.002494362648576498,
238          "observed_residual": 0.0024751098826527596,
239          "intervention": 0.011707877740263939,
240          "test_mse": 0.004916135687381029
241        },
242        {
243          "seed": 5,
244          "predicted_ebar": 0.0024386330042034388,
245          "observed_residual": 0.002423582598567009,
246          "intervention": 0.004593744874000549,
247          "test_mse": 0.002345199463889003
248        },
249        {
250          "seed": 6,
251          "predicted_ebar": 0.002432666253298521,
252          "observed_residual": 0.002475534798577428,
253          "intervention": 0.005940093658864498,
254          "test_mse": 0.004579263739287853
255        },
256        {
257          "seed": 7,
258          "predicted_ebar": 0.002408114029094577,
259          "observed_residual": 0.002417274285107851,
260          "intervention": 0.008047887124121189,
261          "test_mse": 0.0042515830136835575
262        }
263      ],
264      "confirmed": true
265    }
266  }
267}