1{
  2  "bench_version": 1,
  3  "track": "dynamics",
  4  "model": "rnn_small",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "best_cfg": {
  9      "lr": 0.006,
 10      "epochs": 18,
 11      "rho": 0.9,
 12      "k": 2
 13    },
 14    "sweep": [
 15      {
 16        "cfg": {
 17          "lr": 0.001,
 18          "epochs": 18,
 19          "rho": 0.9,
 20          "k": 2
 21        },
 22        "mean": 0.0026806407840922475
 23      },
 24      {
 25        "cfg": {
 26          "lr": 0.003,
 27          "epochs": 18,
 28          "rho": 0.9,
 29          "k": 2
 30        },
 31        "mean": 0.001453446806408465
 32      },
 33      {
 34        "cfg": {
 35          "lr": 0.006,
 36          "epochs": 18,
 37          "rho": 0.9,
 38          "k": 2
 39        },
 40        "mean": 0.0007141287642298266
 41      }
 42    ],
 43    "full": {
 44      "mean": 0.0007108385761966929,
 45      "std": 0.00020260791778280052,
 46      "per_seed": [
 47        0.0008331689168699086,
 48        0.0007008879329077899,
 49        0.0006333848577924073,
 50        0.0008832181920297444,
 51        0.0010298240231350064,
 52        0.0005573644884862006,
 53        0.00032093568006530404,
 54        0.0007279245182871819
 55      ],
 56      "n": 8
 57    }
 58  },
 59  "idea": {
 60    "mean": 0.0006244764626899268,
 61    "std": 0.00017259395969689844,
 62    "per_seed": [
 63      0.0005345472018234432,
 64      0.0004471810825634748,
 65      0.0004362085019238293,
 66      0.0008414528565481305,
 67      0.00048033404164016247,
 68      0.0008765239617787302,
 69      0.0005815455224364996,
 70      0.0007980185328051448
 71    ],
 72    "n": 8
 73  },
 74  "comparison": {
 75    "delta_mean": -8.636211350676604e-05,
 76    "idea_wins": 5,
 77    "n_pairs": 8,
 78    "per_seed_diffs": [
 79      -0.0002986217150464654,
 80      -0.00025370685034431517,
 81      -0.00019717635586857796,
 82      -4.1765335481613874e-05,
 83      -0.000549489981494844,
 84      0.0003191594732925296,
 85      0.00026060984237119555,
 86      7.009401451796293e-05
 87    ],
 88    "p_value": 0.47205,
 89    "mde": 0.0002470609130160734,
 90    "mde_rel_pct": 34.75626130730844,
 91    "verdict": "no significant win",
 92    "system_worked": false
 93  },
 94  "sweep_parity": {
 95    "union_grid": [
 96      {
 97        "lr": 0.001,
 98        "epochs": 18,
 99        "rho": 0.9,
100        "k": 2
101      },
102      {
103        "lr": 0.003,
104        "epochs": 18,
105        "rho": 0.9,
106        "k": 2
107      },
108      {
109        "lr": 0.006,
110        "epochs": 18,
111        "rho": 0.9,
112        "k": 2
113      }
114    ],
115    "idea_sweep": [
116      {
117        "cfg": {
118          "lr": 0.001,
119          "epochs": 18,
120          "rho": 0.9,
121          "k": 2
122        },
123        "mean": 0.0019245385192334652
124      },
125      {
126        "cfg": {
127          "lr": 0.003,
128          "epochs": 18,
129          "rho": 0.9,
130          "k": 2
131        },
132        "mean": 0.0012154859578004107
133      },
134      {
135        "cfg": {
136          "lr": 0.006,
137          "epochs": 18,
138          "rho": 0.9,
139          "k": 2
140        },
141        "mean": 0.0005648474107147194
142      }
143    ]
144  },
145  "adaptation": {
146    "rho": 0.9,
147    "k": 2,
148    "probe_budget": "one batch per epoch"
149  },
150  "mechanism_signature": {
151    "claim": "discounted probe gap is lower late than over training on trained dynamics models",
152    "predicted_late_gap_less_than_mean": true,
153    "observed_mean_gap": 0.0,
154    "observed_late_gap": 0.0,
155    "relative_change": 0.0,
156    "trained_model_measurements": [
157      {
158        "seed": 0,
159        "gap_mean": 0.0,
160        "gap_last": 0.0,
161        "final_lr": 0.006,
162        "down_events": 0,
163        "test_mse": 0.0005345472018234432
164      },
165      {
166        "seed": 1,
167        "gap_mean": 0.0,
168        "gap_last": 0.0,
169        "final_lr": 0.006,
170        "down_events": 0,
171        "test_mse": 0.0004471810825634748
172      },
173      {
174        "seed": 2,
175        "gap_mean": 0.0,
176        "gap_last": 0.0,
177        "final_lr": 0.006,
178        "down_events": 0,
179        "test_mse": 0.0004362085019238293
180      },
181      {
182        "seed": 3,
183        "gap_mean": 0.0,
184        "gap_last": 0.0,
185        "final_lr": 0.006,
186        "down_events": 0,
187        "test_mse": 0.0008414528565481305
188      },
189      {
190        "seed": 4,
191        "gap_mean": 0.0,
192        "gap_last": 0.0,
193        "final_lr": 0.006,
194        "down_events": 0,
195        "test_mse": 0.00048033404164016247
196      },
197      {
198        "seed": 5,
199        "gap_mean": 0.0,
200        "gap_last": 0.0,
201        "final_lr": 0.006,
202        "down_events": 0,
203        "test_mse": 0.0008765239617787302
204      },
205      {
206        "seed": 6,
207        "gap_mean": 0.0,
208        "gap_last": 0.0,
209        "final_lr": 0.006,
210        "down_events": 0,
211        "test_mse": 0.0005815455224364996
212      },
213      {
214        "seed": 7,
215        "gap_mean": 0.0,
216        "gap_last": 0.0,
217        "final_lr": 0.006,
218        "down_events": 0,
219        "test_mse": 0.0007980185328051448
220      }
221    ],
222    "confirmed": false
223  }
224}