1{
  2  "bench_version": 1,
  3  "track": "dynamics",
  4  "model": "rnn_small",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "sweep": {
  9      "grid": [
 10        {
 11          "lr": 0.001,
 12          "epochs": 15
 13        },
 14        {
 15          "lr": 0.003,
 16          "epochs": 15
 17        },
 18        {
 19          "lr": 0.006,
 20          "epochs": 15
 21        }
 22      ],
 23      "results": [
 24        {
 25          "lr": 0.001,
 26          "mean": 0.004110507725272328
 27        },
 28        {
 29          "lr": 0.003,
 30          "mean": 0.0016934662853600457
 31        },
 32        {
 33          "lr": 0.006,
 34          "mean": 0.001336116241873242
 35        }
 36      ],
 37      "best": {
 38        "lr": 0.006,
 39        "mean": 0.001336116241873242
 40      }
 41    },
 42    "full": {
 43      "per_seed": [
 44        0.0008500798139721155,
 45        0.0017577288672327995,
 46        0.0007361507159657776,
 47        0.002000505570322275,
 48        0.0011651096865534782,
 49        0.0009040328441187739,
 50        0.0011707661906257272,
 51        0.0016706058522686362
 52      ],
 53      "records": [
 54        {
 55          "seed": 0,
 56          "lr": 0.006,
 57          "metric": 0.0008500798139721155
 58        },
 59        {
 60          "seed": 1,
 61          "lr": 0.006,
 62          "metric": 0.0017577288672327995
 63        },
 64        {
 65          "seed": 2,
 66          "lr": 0.006,
 67          "metric": 0.0007361507159657776
 68        },
 69        {
 70          "seed": 3,
 71          "lr": 0.006,
 72          "metric": 0.002000505570322275
 73        },
 74        {
 75          "seed": 4,
 76          "lr": 0.006,
 77          "metric": 0.0011651096865534782
 78        },
 79        {
 80          "seed": 5,
 81          "lr": 0.006,
 82          "metric": 0.0009040328441187739
 83        },
 84        {
 85          "seed": 6,
 86          "lr": 0.006,
 87          "metric": 0.0011707661906257272
 88        },
 89        {
 90          "seed": 7,
 91          "lr": 0.006,
 92          "metric": 0.0016706058522686362
 93        }
 94      ]
 95    }
 96  },
 97  "idea": {
 98    "sweep": {
 99      "grid": [
100        {
101          "lr": 0.001,
102          "epochs": 15
103        },
104        {
105          "lr": 0.003,
106          "epochs": 15
107        },
108        {
109          "lr": 0.006,
110          "epochs": 15
111        }
112      ],
113      "results": [
114        {
115          "lr": 0.001,
116          "mean": 0.0054137099359650165
117        },
118        {
119          "lr": 0.003,
120          "mean": 0.001696990402706433
121        },
122        {
123          "lr": 0.006,
124          "mean": 0.0013303105588420294
125        }
126      ],
127      "best": {
128        "lr": 0.006,
129        "mean": 0.0013303105588420294
130      }
131    },
132    "per_seed": [
133      0.0007351692765951157,
134      0.0020680308807641268,
135      0.0006238966016098857,
136      0.0018669400596991181,
137      0.0013283524895086884,
138      0.0009672582964412868,
139      0.0013313123490661383,
140      0.0017215245170518756
141    ],
142    "records": [
143      {
144        "seed": 0,
145        "lr": 0.006,
146        "metric": 0.0007351692765951157,
147        "jacobian_row_norm_mean": 0.5119546502828598
148      },
149      {
150        "seed": 1,
151        "lr": 0.006,
152        "metric": 0.0020680308807641268,
153        "jacobian_row_norm_mean": 0.4930251557379961
154      },
155      {
156        "seed": 2,
157        "lr": 0.006,
158        "metric": 0.0006238966016098857,
159        "jacobian_row_norm_mean": 0.5136511726304889
160      },
161      {
162        "seed": 3,
163        "lr": 0.006,
164        "metric": 0.0018669400596991181,
165        "jacobian_row_norm_mean": 0.49356979224830866
166      },
167      {
168        "seed": 4,
169        "lr": 0.006,
170        "metric": 0.0013283524895086884,
171        "jacobian_row_norm_mean": 0.5176679929718375
172      },
173      {
174        "seed": 5,
175        "lr": 0.006,
176        "metric": 0.0009672582964412868,
177        "jacobian_row_norm_mean": 0.5031084371730685
178      },
179      {
180        "seed": 6,
181        "lr": 0.006,
182        "metric": 0.0013313123490661383,
183        "jacobian_row_norm_mean": 0.5190566899254918
184      },
185      {
186        "seed": 7,
187        "lr": 0.006,
188        "metric": 0.0017215245170518756,
189        "jacobian_row_norm_mean": 0.5010124798864126
190      }
191    ]
192  },
193  "comparison": {
194    "delta_mean": 4.8438116209581494e-05,
195    "idea_wins": 3,
196    "n_pairs": 8,
197    "per_seed_diffs": [
198      -0.00011491053737699986,
199      0.00031030201353132725,
200      -0.00011225411435589194,
201      -0.00013356551062315702,
202      0.0001632428029552102,
203      6.322545232251287e-05,
204      0.0001605461584404111,
205      5.0918664783239365e-05
206    ],
207    "p_value": 0.43475,
208    "mde": 0.000134148411633326,
209    "mde_rel_pct": 10.465035924934886,
210    "verdict": "no significant win",
211    "system_worked": false
212  },
213  "mechanism_signature": {
214    "prediction": "greedy delay taps have larger observability margin than uniform taps at equal budget",
215    "predicted_margin_greedy": 0.2985928640520014,
216    "predicted_margin_uniform": 0.26431666900098605,
217    "selected_indices": [
218      0,
219      1,
220      6,
221      7
222    ],
223    "uniform_indices": [
224      0,
225      2,
226      5,
227      7
228    ],
229    "trained_model_observed": {
230      "idea_test_mse_mean": 0.0013303105588420294,
231      "idea_jacobian_row_norm_mean": 0.506630796357058
232    },
233    "confirmed": true
234  }
235}