1{
  2  "bench_version": 1,
  3  "track": "dynamics",
  4  "model": "rnn_small",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "best_cfg": {
  9      "lr": 0.003,
 10      "epochs": 12
 11    },
 12    "sweep": [
 13      {
 14        "cfg": {
 15          "lr": 0.0015,
 16          "epochs": 12
 17        },
 18        "mean": 0.03892233967781067
 19      },
 20      {
 21        "cfg": {
 22          "lr": 0.003,
 23          "epochs": 12
 24        },
 25        "mean": 0.030090928077697754
 26      },
 27      {
 28        "cfg": {
 29          "lr": 0.006,
 30          "epochs": 12
 31        },
 32        "mean": 0.05556618981063366
 33      }
 34    ],
 35    "full": {
 36      "mean": 0.030090928077697754,
 37      "std": 0.004071433001350898,
 38      "per_seed": [
 39        0.03666028752923012,
 40        0.02726038359105587,
 41        0.030306316912174225,
 42        0.03209355100989342,
 43        0.024804387241601944,
 44        0.033685918897390366,
 45        0.024255627766251564,
 46        0.03166095167398453
 47      ],
 48      "n": 8
 49    },
 50    "searched_grid": [
 51      {
 52        "lr": 0.0015,
 53        "epochs": 12
 54      },
 55      {
 56        "lr": 0.003,
 57        "epochs": 12
 58      },
 59      {
 60        "lr": 0.006,
 61        "epochs": 12
 62      }
 63    ]
 64  },
 65  "idea": {
 66    "mean": 0.03774771373718977,
 67    "std": 0.004092688612850228,
 68    "per_seed": [
 69      0.044951874762773514,
 70      0.037032078951597214,
 71      0.0381438285112381,
 72      0.04309968277812004,
 73      0.03429986909031868,
 74      0.037081118673086166,
 75      0.03175382688641548,
 76      0.035619430243968964
 77    ],
 78    "n": 8,
 79    "best_config": {
 80      "lr": 0.003,
 81      "epochs": 12
 82    }
 83  },
 84  "comparison": {
 85    "delta_mean": 0.007656785659492016,
 86    "idea_wins": 0,
 87    "n_pairs": 8,
 88    "per_seed_diffs": [
 89      0.008291587233543396,
 90      0.009771695360541344,
 91      0.007837511599063873,
 92      0.011006131768226624,
 93      0.009495481848716736,
 94      0.0033951997756958008,
 95      0.0074981991201639175,
 96      0.003958478569984436
 97    ],
 98    "p_value": 0.0081,
 99    "mde": 0.0022651487884256674,
100    "mde_rel_pct": 7.527680045550038,
101    "verdict": "idea worse (significant)",
102    "system_worked": false
103  },
104  "mechanism_signature": {
105    "predicted": {
106      "unsafe_fraction": "elevated near mixed signed threshold",
107      "order_effect": "nonzero"
108    },
109    "observed": {
110      "high_drive_fraction": 0.25,
111      "mean_output_abs_difference": 0.15396859403699636
112    },
113    "confirmed": true
114  },
115  "idea_sweep": {
116    "{'lr': 0.0015, 'epochs': 12}": {
117      "mean": 0.05525698186829686,
118      "std": 0.02605631177795089,
119      "per_seed": [
120        0.06213795393705368,
121        0.06313511729240417,
122        0.04352016746997833,
123        0.11799561977386475,
124        0.03681809827685356,
125        0.03047773241996765,
126        0.03951030224561691,
127        0.048460863530635834
128      ],
129      "n": 8
130    },
131    "{'lr': 0.003, 'epochs': 12}": {
132      "mean": 0.03774771373718977,
133      "std": 0.004092688612850228,
134      "per_seed": [
135        0.044951874762773514,
136        0.037032078951597214,
137        0.0381438285112381,
138        0.04309968277812004,
139        0.03429986909031868,
140        0.037081118673086166,
141        0.03175382688641548,
142        0.035619430243968964
143      ],
144      "n": 8,
145      "best_config": {
146        "lr": 0.003,
147        "epochs": 12
148      }
149    },
150    "{'lr': 0.006, 'epochs': 12}": {
151      "mean": 0.041616966016590595,
152      "std": 0.0063707081731607254,
153      "per_seed": [
154        0.044290054589509964,
155        0.04693755879998207,
156        0.03771070018410683,
157        0.0332266241312027,
158        0.036028288304805756,
159        0.04784112051129341,
160        0.035485830157995224,
161        0.05141555145382881
162      ],
163      "n": 8
164    }
165  },
166  "protocol": {
167    "seeds": [
168      0,
169      1,
170      2,
171      3,
172      4,
173      5,
174      6,
175      7
176    ],
177    "n_train": 400,
178    "n_test": 400,
179    "matched_architecture": true,
180    "shared_hyperparameter_union": [
181      {
182        "lr": 0.0015,
183        "epochs": 12
184      },
185      {
186        "lr": 0.003,
187        "epochs": 12
188      },
189      {
190        "lr": 0.006,
191        "epochs": 12
192      }
193    ]
194  }
195}