1{
  2  "bench_version": 1,
  3  "track": "masked_multitoken_modes",
  4  "model": "transformer_tiny",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "best_cfg": {
  9      "lr": 0.002,
 10      "epochs": 14,
 11      "hide_count": 1
 12    },
 13    "sweep": [
 14      {
 15        "cfg": {
 16          "lr": 0.002,
 17          "epochs": 14,
 18          "hide_count": 1
 19        },
 20        "mean": 0.1428266018629074
 21      },
 22      {
 23        "cfg": {
 24          "lr": 0.004,
 25          "epochs": 14,
 26          "hide_count": 1
 27        },
 28        "mean": 0.20178561471402645
 29      },
 30      {
 31        "cfg": {
 32          "lr": 0.008,
 33          "epochs": 14,
 34          "hide_count": 1
 35        },
 36        "mean": 0.14775850623846054
 37      }
 38    ],
 39    "full": {
 40      "mean": 0.14373092539608479,
 41      "std": 0.02385596096955299,
 42      "per_seed": [
 43        0.1670890897512436,
 44        0.12829364836215973,
 45        0.13317705690860748,
 46        0.14274661242961884,
 47        0.16149330139160156,
 48        0.09596626460552216,
 49        0.14481675624847412,
 50        0.1762646734714508
 51      ],
 52      "n": 8
 53    }
 54  },
 55  "idea": {
 56    "mean": 0.14993123710155487,
 57    "std": 0.038141062047303144,
 58    "per_seed": [
 59      0.1759413331747055,
 60      0.13947612047195435,
 61      0.12913769483566284,
 62      0.14234721660614014,
 63      0.149277925491333,
 64      0.09132890403270721,
 65      0.13944129645824432,
 66      0.2324994057416916
 67    ],
 68    "n": 8,
 69    "best_cfg": {
 70      "lr": 0.004,
 71      "epochs": 14,
 72      "lam": 0.1,
 73      "rho": 0.01
 74    }
 75  },
 76  "comparison": {
 77    "delta_mean": 0.006200311705470085,
 78    "idea_wins": 5,
 79    "n_pairs": 8,
 80    "per_seed_diffs": [
 81      0.008852243423461914,
 82      0.011182472109794617,
 83      -0.004039362072944641,
 84      -0.00039939582347869873,
 85      -0.012215375900268555,
 86      -0.004637360572814941,
 87      -0.005375459790229797,
 88      0.056234732270240784
 89    ],
 90    "p_value": 0.63155,
 91    "mde": 0.018084067093522574,
 92    "mde_rel_pct": 12.581890114244809,
 93    "verdict": "no significant win",
 94    "system_worked": false
 95  },
 96  "mechanism_signature": {
 97    "schedule_check": [
 98      {
 99        "lambda": 0,
100        "rho": 0,
101        "pred_pi1": 0,
102        "obs_pi1": 0.0,
103        "pred_full": 0,
104        "obs_full": 0.0,
105        "v_min": 7
106      },
107      {
108        "lambda": 0.1,
109        "rho": 0,
110        "pred_pi1": 0.1,
111        "obs_pi1": 0.10140000283718109,
112        "pred_full": 0,
113        "obs_full": 0.0,
114        "v_min": 0
115      },
116      {
117        "lambda": 0.1,
118        "rho": 0.01,
119        "pred_pi1": 0.11,
120        "obs_pi1": 0.11155000329017639,
121        "pred_full": 0.01,
122        "obs_full": 0.009879999794065952,
123        "v_min": 0
124      },
125      {
126        "lambda": 0.25,
127        "rho": 0.01,
128        "pred_pi1": 0.26,
129        "obs_pi1": 0.26243001222610474,
130        "pred_full": 0.01,
131        "obs_full": 0.009879999794065952,
132        "v_min": 0
133      }
134    ],
135    "signature": {
136      "quantity": "empty-context mode probability",
137      "target_true_weight": 0.8,
138      "predicted_baseline": 0.8209748044610023,
139      "predicted_idea": 0.839116208255291,
140      "observed_baseline": 0.8209748044610023,
141      "observed_idea": 0.839116208255291,
142      "predicted_improvement": -0.018141403794288635,
143      "confirmed": false
144    },
145    "custom_track": {
146      "name": "masked_multitoken_modes",
147      "file": "custom_masked_track.py",
148      "domain": "sequence"
149    }
150  },
151  "idea_sweep": [
152    {
153      "cfg": {
154        "lr": 0.002,
155        "epochs": 14,
156        "lam": 0.1,
157        "rho": 0.0
158      },
159      "mean": 0.16454514022916555,
160      "per_seed": [
161        0.16334068775177002,
162        0.22535602748394012,
163        0.12713226675987244,
164        0.18480685353279114,
165        0.19457480311393738,
166        0.10892865806818008,
167        0.14529073238372803,
168        0.16693109273910522
169      ],
170      "mode_probs": [
171        0.7183287739753723,
172        0.8767662644386292,
173        0.9214068055152893,
174        0.8836508393287659,
175        0.8848533034324646,
176        0.7462123036384583,
177        0.780626654624939,
178        0.8711745738983154
179      ]
180    },
181    {
182      "cfg": {
183        "lr": 0.004,
184        "epochs": 14,
185        "lam": 0.1,
186        "rho": 0.01
187      },
188      "mean": 0.14993123710155487,
189      "per_seed": [
190        0.1759413331747055,
191        0.13947612047195435,
192        0.12913769483566284,
193        0.14234721660614014,
194        0.149277925491333,
195        0.09132890403270721,
196        0.13944129645824432,
197        0.2324994057416916
198      ],
199      "mode_probs": [
200        0.9315788745880127,
201        0.8918474316596985,
202        0.8515940308570862,
203        0.9011403918266296,
204        0.7730483412742615,
205        0.7809845209121704,
206        0.7969619035720825,
207        0.7857741713523865
208      ]
209    },
210    {
211      "cfg": {
212        "lr": 0.008,
213        "epochs": 14,
214        "lam": 0.25,
215        "rho": 0.01
216      },
217      "mean": 0.18514209240674973,
218      "per_seed": [
219        0.18203185498714447,
220        0.16977907717227936,
221        0.13093167543411255,
222        0.2541387677192688,
223        0.21239057183265686,
224        0.16110031306743622,
225        0.14696472883224487,
226        0.22379975020885468
227      ],
228      "mode_probs": [
229        0.7211427092552185,
230        0.8639533519744873,
231        0.8087149262428284,
232        0.8416880965232849,
233        0.8448388576507568,
234        0.8393517136573792,
235        0.7988701462745667,
236        0.8555705547332764
237      ]
238    }
239  ]
240}