1{
  2  "bench_version": 1,
  3  "track": "dynamics",
  4  "model": "rnn_small",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "best_cfg": {
  9      "lr": 0.01
 10    },
 11    "sweep": [
 12      {
 13        "cfg": {
 14          "lr": 0.001
 15        },
 16        "mean": 0.0018504196486901492
 17      },
 18      {
 19        "cfg": {
 20          "lr": 0.003
 21        },
 22        "mean": 0.00105226032610517
 23      },
 24      {
 25        "cfg": {
 26          "lr": 0.01
 27        },
 28        "mean": 0.0005928000318817794
 29      }
 30    ],
 31    "full": {
 32      "per_seed": [
 33        0.000473822612548247,
 34        0.0005578889395110309,
 35        0.00041418071486987174,
 36        0.0009253078605979681,
 37        0.0004536386113613844,
 38        0.0002683700295165181,
 39        0.0006777027156203985,
 40        0.000613455253187567
 41      ],
 42      "mean": 0.0005480458421516232,
 43      "std": 0.00019787191330725143
 44    }
 45  },
 46  "idea": {
 47    "per_seed": [
 48      0.06354859471321106,
 49      0.07997369021177292,
 50      0.08213100582361221,
 51      0.06680075824260712,
 52      0.07302123308181763,
 53      0.06840986758470535,
 54      0.06384869664907455,
 55      0.07170670479536057
 56    ],
 57    "mean": 0.07118006888777018,
 58    "std": 0.006966848239001737,
 59    "sweep": [
 60      {
 61        "cfg": {
 62          "lr": 0.001,
 63          "kappa": 0.25
 64        },
 65        "mean": 0.07311351224780083
 66      },
 67      {
 68        "cfg": {
 69          "lr": 0.001,
 70          "kappa": 0.5
 71        },
 72        "mean": 0.11853109858930111
 73      },
 74      {
 75        "cfg": {
 76          "lr": 0.001,
 77          "kappa": 1.0
 78        },
 79        "mean": 0.24801216274499893
 80      },
 81      {
 82        "cfg": {
 83          "lr": 0.003,
 84          "kappa": 0.25
 85        },
 86        "mean": 0.08233587071299553
 87      },
 88      {
 89        "cfg": {
 90          "lr": 0.003,
 91          "kappa": 0.5
 92        },
 93        "mean": 0.10741026885807514
 94      },
 95      {
 96        "cfg": {
 97          "lr": 0.003,
 98          "kappa": 1.0
 99        },
100        "mean": 0.17653513327240944
101      },
102      {
103        "cfg": {
104          "lr": 0.01,
105          "kappa": 0.25
106        },
107        "mean": 0.0779394619166851
108      },
109      {
110        "cfg": {
111          "lr": 0.01,
112          "kappa": 0.5
113        },
114        "mean": 0.10071836225688457
115      },
116      {
117        "cfg": {
118          "lr": 0.01,
119          "kappa": 1.0
120        },
121        "mean": 0.23662623763084412
122      }
123    ],
124    "best_cfg": {
125      "lr": 0.001,
126      "kappa": 0.25
127    }
128  },
129  "comparison": {
130    "delta_mean": 0.07063202304561855,
131    "idea_wins": 0,
132    "n_pairs": 8,
133    "per_seed_diffs": [
134      0.06307477210066281,
135      0.07941580127226189,
136      0.08171682510874234,
137      0.06587545038200915,
138      0.07256759447045624,
139      0.06814149755518883,
140      0.06317099393345416,
141      0.071093249542173
142    ],
143    "p_value": 0.0081,
144    "mde": 0.005875162592988052,
145    "mde_rel_pct": 1072.0202839095018,
146    "verdict": "idea worse (significant)",
147    "system_worked": false
148  },
149  "mechanism_signature": {
150    "per_seed": [
151      {
152        "seed": 0,
153        "observed_residual": 0.2520884573459625,
154        "normalized_residual": 0.7100334684505998,
155        "tightening_margin": 0.17750836711264995,
156        "predicted_safe_fraction": 0.98,
157        "observed_safe_fraction": 0.9
158      },
159      {
160        "seed": 1,
161        "observed_residual": 0.2827962040901184,
162        "normalized_residual": 0.7726593654177988,
163        "tightening_margin": 0.1931648413544497,
164        "predicted_safe_fraction": 0.9675,
165        "observed_safe_fraction": 0.865
166      },
167      {
168        "seed": 2,
169        "observed_residual": 0.2865850627422333,
170        "normalized_residual": 0.7903462817844339,
171        "tightening_margin": 0.19758657044610847,
172        "predicted_safe_fraction": 0.98,
173        "observed_safe_fraction": 0.9025
174      },
175      {
176        "seed": 3,
177        "observed_residual": 0.2584584355354309,
178        "normalized_residual": 0.7446077562600646,
179        "tightening_margin": 0.18615193906501615,
180        "predicted_safe_fraction": 0.98,
181        "observed_safe_fraction": 0.9
182      },
183      {
184        "seed": 4,
185        "observed_residual": 0.2702244222164154,
186        "normalized_residual": 0.7408200912064927,
187        "tightening_margin": 0.18520502280162318,
188        "predicted_safe_fraction": 0.98,
189        "observed_safe_fraction": 0.8825
190      },
191      {
192        "seed": 5,
193        "observed_residual": 0.2615528106689453,
194        "normalized_residual": 0.7201894427650053,
195        "tightening_margin": 0.18004736069125132,
196        "predicted_safe_fraction": 0.995,
197        "observed_safe_fraction": 0.895
198      },
199      {
200        "seed": 6,
201        "observed_residual": 0.25268301367759705,
202        "normalized_residual": 0.7071742747233145,
203        "tightening_margin": 0.17679356868082863,
204        "predicted_safe_fraction": 0.9825,
205        "observed_safe_fraction": 0.9025
206      },
207      {
208        "seed": 7,
209        "observed_residual": 0.2677810788154602,
210        "normalized_residual": 0.7319731444569756,
211        "tightening_margin": 0.1829932861142439,
212        "predicted_safe_fraction": 0.9875,
213        "observed_safe_fraction": 0.8975
214      }
215    ],
216    "residual_margin_correlation": 0.9318449762336007,
217    "prediction": "higher trained-model residual produces higher tightening margin",
218    "confirmed": true
219  }
220}