1{
  2  "bench_version": 1,
  3  "track": "dynamics",
  4  "model": "rnn_small",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "best_cfg": {
  9      "lr": 0.00665
 10    },
 11    "sweep": [
 12      {
 13        "cfg": {
 14          "lr": 0.001
 15        },
 16        "mean": 0.0009722943214001134
 17      },
 18      {
 19        "cfg": {
 20          "lr": 0.002
 21        },
 22        "mean": 0.0006509074009954929
 23      },
 24      {
 25        "cfg": {
 26          "lr": 0.003
 27        },
 28        "mean": 0.0004595852587954141
 29      },
 30      {
 31        "cfg": {
 32          "lr": 0.004
 33        },
 34        "mean": 0.0003482927131699398
 35      },
 36      {
 37        "cfg": {
 38          "lr": 0.005
 39        },
 40        "mean": 0.00024859366021701135
 41      },
 42      {
 43        "cfg": {
 44          "lr": 0.00067
 45        },
 46        "mean": 0.0011799605708802119
 47      },
 48      {
 49        "cfg": {
 50          "lr": 0.00134
 51        },
 52        "mean": 0.0008329209085786715
 53      },
 54      {
 55        "cfg": {
 56          "lr": 0.00268
 57        },
 58        "mean": 0.0005083113646833226
 59      },
 60      {
 61        "cfg": {
 62          "lr": 0.00402
 63        },
 64        "mean": 0.0003466853668214753
 65      },
 66      {
 67        "cfg": {
 68          "lr": 0.00536
 69        },
 70        "mean": 0.00022448933304985985
 71      },
 72      {
 73        "cfg": {
 74          "lr": 0.00665
 75        },
 76        "mean": 0.00013951877372164745
 77      }
 78    ],
 79    "full": {
 80      "mean": 0.00011474007897049887,
 81      "std": 4.026902293849407e-05,
 82      "per_seed": [
 83        0.0001932993473019451,
 84        0.0001438440231140703,
 85        8.013794285943732e-05,
 86        0.0001407937816111371,
 87        0.00011875866039190441,
 88        8.25471433927305e-05,
 89        6.331295298878103e-05,
 90        9.522678010398522e-05
 91      ],
 92      "n": 8
 93    }
 94  },
 95  "idea": {
 96    "mean": 0.05237803841009736,
 97    "std": 0.008875649004641814,
 98    "per_seed": [
 99      0.04106063395738602,
100      0.058054666966199875,
101      0.0539846234023571,
102      0.06823215633630753,
103      0.05530509725213051,
104      0.03805254027247429,
105      0.05319170653820038,
106      0.05114288255572319
107    ],
108    "n": 8
109  },
110  "comparison": {
111    "delta_mean": 0.05226329833112686,
112    "idea_wins": 0,
113    "n_pairs": 8,
114    "per_seed_diffs": [
115      0.04086733461008407,
116      0.057910822943085805,
117      0.053904485459497664,
118      0.06809136255469639,
119      0.055186338591738604,
120      0.03796999312908156,
121      0.0531283935852116,
122      0.051047655775619205
123    ],
124    "p_value": 0.0081,
125    "mde": 0.007932781978214037,
126    "mde_rel_pct": 6913.697506042031,
127    "verdict": "idea worse (significant)",
128    "system_worked": false
129  },
130  "mechanism_signature": {
131    "farkas_shield": {
132      "constraint_count_full": 64,
133      "constraint_count_retained": 2,
134      "retained_fraction": 0.03125,
135      "certificate_max_residual": 0.0,
136      "certificate_max_offset_violation": 0.0,
137      "trained_test_samples": 400,
138      "full_reduced_max_output_difference": 0.0,
139      "full_reduced_decision_disagreement": 0.0,
140      "predicted_zero_disagreement_observed": true,
141      "confirmed": true
142    }
143  },
144  "idea_sweep": [
145    {
146      "cfg": {
147        "lr": 0.00665
148      },
149      "result": {
150        "mean": 0.05238204402849078,
151        "std": 0.008873981965744437,
152        "per_seed": [
153          0.04106547310948372,
154          0.0580538772046566,
155          0.0539894662797451,
156          0.06823457032442093,
157          0.055308111011981964,
158          0.03805993124842644,
159          0.053193967789411545,
160          0.05115095525979996
161        ],
162        "n": 8
163      }
164    },
165    {
166      "cfg": {
167        "lr": 0.0044555
168      },
169      "result": {
170        "mean": 0.05238870158791542,
171        "std": 0.008873361924843962,
172        "per_seed": [
173          0.04107039421796799,
174          0.0580560676753521,
175          0.053996019065380096,
176          0.06824300438165665,
177          0.055311888456344604,
178          0.03807036951184273,
179          0.053200963884592056,
180          0.05116090551018715
181        ],
182        "n": 8
183      }
184    },
185    {
186      "cfg": {
187        "lr": 0.0088445
188      },
189      "result": {
190        "mean": 0.05237803841009736,
191        "std": 0.008875649004641814,
192        "per_seed": [
193          0.04106063395738602,
194          0.058054666966199875,
195          0.0539846234023571,
196          0.06823215633630753,
197          0.05530509725213051,
198          0.03805254027247429,
199          0.05319170653820038,
200          0.05114288255572319
201        ],
202        "n": 8
203      }
204    }
205  ],
206  "protocol_notes": {
207    "task_match": "controlled pendulum dynamics; safety/control idea",
208    "epochs": 15,
209    "n_train": 1200,
210    "n_test": 400,
211    "idea_uses_same_base_architecture": true
212  }
213}