1{
  2  "bench_version": 1,
  3  "track": "dynamics",
  4  "model": "rnn_small",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "best_cfg": {
  9      "lr": 0.006,
 10      "epochs": 10,
 11      "branches": 1
 12    },
 13    "sweep": [
 14      {
 15        "cfg": {
 16          "lr": 0.001,
 17          "epochs": 10,
 18          "branches": 1
 19        },
 20        "mean": 0.0019458123424556106
 21      },
 22      {
 23        "cfg": {
 24          "lr": 0.003,
 25          "epochs": 10,
 26          "branches": 1
 27        },
 28        "mean": 0.000998065312160179
 29      },
 30      {
 31        "cfg": {
 32          "lr": 0.006,
 33          "epochs": 10,
 34          "branches": 1
 35        },
 36        "mean": 0.0007849872272345237
 37      }
 38    ],
 39    "full": {
 40      "mean": 0.0007333773064601701,
 41      "std": 0.00023119481143299165,
 42      "per_seed": [
 43        0.0007082645315676928,
 44        0.0011445400305092335,
 45        0.00047928086132742465,
 46        0.0008078634855337441,
 47        0.00035395112354308367,
 48        0.0006696216878481209,
 49        0.0007750760996714234,
 50        0.0009284206316806376
 51      ],
 52      "n": 8
 53    }
 54  },
 55  "idea": {
 56    "cfg": {
 57      "lr": 0.006,
 58      "epochs": 10,
 59      "branches": 5
 60    },
 61    "mean": 0.0008847052595228888,
 62    "std": 0.00026667546357193395,
 63    "per_seed": [
 64      0.0010930252028629184,
 65      0.0010985390981659293,
 66      0.0007570339366793633,
 67      0.001170001458376646,
 68      0.00035861978540197015,
 69      0.0010039987973868847,
 70      0.0006030820077285171,
 71      0.0009933417895808816
 72    ],
 73    "n": 8
 74  },
 75  "comparison": {
 76    "delta_mean": 0.00015132795306271873,
 77    "idea_wins": 2,
 78    "n_pairs": 8,
 79    "per_seed_diffs": [
 80      0.0003847606712952256,
 81      -4.600093234330416e-05,
 82      0.0002777530753519386,
 83      0.00036213797284290195,
 84      4.66866185888648e-06,
 85      0.00033437710953876376,
 86      -0.00017199409194290638,
 87      6.4921157900244e-05
 88    ],
 89    "p_value": 0.08585,
 90    "mde": 0.00017902087041145347,
 91    "mde_rel_pct": 24.410473140427907,
 92    "verdict": "no significant win",
 93    "system_worked": false
 94  },
 95  "idea_sweep": [
 96    {
 97      "cfg": {
 98        "lr": 0.001,
 99        "epochs": 10,
100        "branches": 5
101      },
102      "mean": 0.00476446797256358,
103      "std": 0.0010113119245195041,
104      "per_seed": [
105        0.004423358011990786,
106        0.00587823148816824,
107        0.006476645823568106,
108        0.0045180427841842175,
109        0.0030923711601644754,
110        0.004275563172996044,
111        0.005356233566999435,
112        0.004095297772437334
113      ],
114      "n": 8
115    },
116    {
117      "cfg": {
118        "lr": 0.003,
119        "epochs": 10,
120        "branches": 5
121      },
122      "mean": 0.0016875666624400765,
123      "std": 0.00044316453275384946,
124      "per_seed": [
125        0.0008459890959784389,
126        0.0019791238009929657,
127        0.0021841360721737146,
128        0.001931163715198636,
129        0.0013305676402524114,
130        0.0013080938952043653,
131        0.0018005802994593978,
132        0.002120878780260682
133      ],
134      "n": 8
135    },
136    {
137      "cfg": {
138        "lr": 0.006,
139        "epochs": 10,
140        "branches": 5
141      },
142      "mean": 0.0008847052595228888,
143      "std": 0.00026667546357193395,
144      "per_seed": [
145        0.0010930252028629184,
146        0.0010985390981659293,
147        0.0007570339366793633,
148        0.001170001458376646,
149        0.00035861978540197015,
150        0.0010039987973868847,
151        0.0006030820077285171,
152        0.0009933417895808816
153      ],
154      "n": 8
155    }
156  ],
157  "budget": {
158    "n_train": 800,
159    "n_test": 300,
160    "epochs": 10,
161    "batch": 128,
162    "lr_union": [
163      0.001,
164      0.003,
165      0.006
166    ]
167  },
168  "mechanism_signature": {
169    "trained_model": true,
170    "soft_minus_hard_mean": 0.004501420073211193,
171    "soft_minus_hard_std": 0.008464416489005089,
172    "branch_spread_mean": 0.2787400186061859,
173    "subset_refinement": [
174      {
175        "M": 2,
176        "mean_abs_to_full": 0.5144225358963013
177      },
178      {
179        "M": 3,
180        "mean_abs_to_full": 0.035924121737480164
181      },
182      {
183        "M": 5,
184        "mean_abs_to_full": 0.0
185      }
186    ],
187    "confirmed": true,
188    "note": "values measured on the seed-0 trained dynamics model"
189  }
190}