1{
  2  "bench_version": 1,
  3  "track": "dynamics",
  4  "model": "rnn_small",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "best_cfg": {
  9      "lr": 0.01,
 10      "epochs": 12
 11    },
 12    "sweep": [
 13      {
 14        "cfg": {
 15          "lr": 0.001,
 16          "epochs": 12
 17        },
 18        "mean": 0.004103701037820429
 19      },
 20      {
 21        "cfg": {
 22          "lr": 0.003,
 23          "epochs": 12
 24        },
 25        "mean": 0.0033861721749417484
 26      },
 27      {
 28        "cfg": {
 29          "lr": 0.01,
 30          "epochs": 12
 31        },
 32        "mean": 0.0014241317112464458
 33      }
 34    ],
 35    "full": {
 36      "mean": 0.001394080012687482,
 37      "std": 0.0003941947646392458,
 38      "per_seed": [
 39        0.0008339877240359783,
 40        0.0018332540057599545,
 41        0.001035665743984282,
 42        0.0019936193712055683,
 43        0.0009847625624388456,
 44        0.0013270708732306957,
 45        0.0016533465823158622,
 46        0.0014909332385286689
 47      ],
 48      "n": 8
 49    }
 50  },
 51  "idea": {
 52    "mean": 0.0013935517708887346,
 53    "std": 0.0003435936282481101,
 54    "per_seed": [
 55      0.000962425721809268,
 56      0.0018949415534734726,
 57      0.0011497323866933584,
 58      0.0018263080855831504,
 59      0.0009521312895230949,
 60      0.0014383804518729448,
 61      0.0016268157633021474,
 62      0.0012976789148524404
 63    ],
 64    "n": 8
 65  },
 66  "comparison": {
 67    "delta_mean": -5.282417987473309e-07,
 68    "idea_wins": 4,
 69    "n_pairs": 8,
 70    "per_seed_diffs": [
 71      0.00012843799777328968,
 72      6.168754771351814e-05,
 73      0.0001140666427090764,
 74      -0.00016731128562241793,
 75      -3.263127291575074e-05,
 76      0.00011130957864224911,
 77      -2.653081901371479e-05,
 78      -0.00019325432367622852
 79    ],
 80    "p_value": 0.98515,
 81    "mde": 0.0001061475231632145,
 82    "mde_rel_pct": 7.614162902930173,
 83    "verdict": "no measurable effect",
 84    "system_worked": false
 85  },
 86  "mechanism_signature": {
 87    "prediction": "accepted Petri successors remain admissible",
 88    "closure_violations": 0,
 89    "predicted_unsafe_accepted_rate": 0.0,
 90    "observed_unsafe_prediction_rate_trained_idea": 0.07,
 91    "observed_rejection_rate_trained_task_inputs": 0.05347656179219484,
 92    "trained_model_rows": [
 93      {
 94        "seed": 0,
 95        "baseline_metric": 0.0008339877240359783,
 96        "idea_metric": 0.000962425721809268,
 97        "baseline_unsafe_prediction_rate": 0.045,
 98        "idea_unsafe_prediction_rate": 0.0425,
 99        "shield_rejection_rate": 0.03531249985098839
100      },
101      {
102        "seed": 1,
103        "baseline_metric": 0.0018332540057599545,
104        "idea_metric": 0.0018949415534734726,
105        "baseline_unsafe_prediction_rate": 0.095,
106        "idea_unsafe_prediction_rate": 0.0975,
107        "shield_rejection_rate": 0.059687498956918716
108      },
109      {
110        "seed": 2,
111        "baseline_metric": 0.001035665743984282,
112        "idea_metric": 0.0011497323866933584,
113        "baseline_unsafe_prediction_rate": 0.055,
114        "idea_unsafe_prediction_rate": 0.055,
115        "shield_rejection_rate": 0.051874998956918716
116      },
117      {
118        "seed": 3,
119        "baseline_metric": 0.0019936193712055683,
120        "idea_metric": 0.0018263080855831504,
121        "baseline_unsafe_prediction_rate": 0.1225,
122        "idea_unsafe_prediction_rate": 0.1225,
123        "shield_rejection_rate": 0.0793749988079071
124      },
125      {
126        "seed": 4,
127        "baseline_metric": 0.0009847625624388456,
128        "idea_metric": 0.0009521312895230949,
129        "baseline_unsafe_prediction_rate": 0.0725,
130        "idea_unsafe_prediction_rate": 0.07,
131        "shield_rejection_rate": 0.054999999701976776
132      },
133      {
134        "seed": 5,
135        "baseline_metric": 0.0013270708732306957,
136        "idea_metric": 0.0014383804518729448,
137        "baseline_unsafe_prediction_rate": 0.0325,
138        "idea_unsafe_prediction_rate": 0.0325,
139        "shield_rejection_rate": 0.03312499821186066
140      },
141      {
142        "seed": 6,
143        "baseline_metric": 0.0016533465823158622,
144        "idea_metric": 0.0016268157633021474,
145        "baseline_unsafe_prediction_rate": 0.055,
146        "idea_unsafe_prediction_rate": 0.0575,
147        "shield_rejection_rate": 0.05843750014901161
148      },
149      {
150        "seed": 7,
151        "baseline_metric": 0.0014909332385286689,
152        "idea_metric": 0.0012976789148524404,
153        "baseline_unsafe_prediction_rate": 0.0825,
154        "idea_unsafe_prediction_rate": 0.0825,
155        "shield_rejection_rate": 0.054999999701976776
156      }
157    ],
158    "confirmed": true
159  },
160  "idea_sweep": [
161    {
162      "cfg": {
163        "lr": 0.001,
164        "epochs": 12
165      },
166      "result": {
167        "mean": 0.004844218114158139,
168        "std": 0.0021060335026899866,
169        "per_seed": [
170          0.0076133571565151215,
171          0.0035649114288389683,
172          0.001954183680936694,
173          0.002968552988022566,
174          0.00587603310123086,
175          0.004371007438749075,
176          0.00404327642172575,
177          0.008362422697246075
178        ],
179        "n": 8
180      }
181    },
182    {
183      "cfg": {
184        "lr": 0.003,
185        "epochs": 12
186      },
187      "result": {
188        "mean": 0.002841877590981312,
189        "std": 0.0016170159627390552,
190        "per_seed": [
191          0.0020566112361848354,
192          0.0030648154206573963,
193          0.0009700310183688998,
194          0.006774549372494221,
195          0.0028293621726334095,
196          0.002948844339698553,
197          0.002073001815006137,
198          0.002017805352807045
199        ],
200        "n": 8
201      }
202    },
203    {
204      "cfg": {
205        "lr": 0.01,
206        "epochs": 12
207      },
208      "result": {
209        "mean": 0.0013935517708887346,
210        "std": 0.0003435936282481101,
211        "per_seed": [
212          0.000962425721809268,
213          0.0018949415534734726,
214          0.0011497323866933584,
215          0.0018263080855831504,
216          0.0009521312895230949,
217          0.0014383804518729448,
218          0.0016268157633021474,
219          0.0012976789148524404
220        ],
221        "n": 8
222      }
223    }
224  ],
225  "selected_idea_cfg": {
226    "lr": 0.01,
227    "epochs": 12
228  },
229  "track_justification": "dynamics is the matched control/stability benchmark: actuated pendulum rollouts."
230}