1{
  2  "bench_version": 1,
  3  "track": "sequence",
  4  "model": "transformer_tiny",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "best_cfg": {
  9      "lr": 0.004,
 10      "keep": 12
 11    },
 12    "sweep": [
 13      {
 14        "cfg": {
 15          "lr": 0.002,
 16          "keep": 4
 17        },
 18        "mean": 0.7055840939283371
 19      },
 20      {
 21        "cfg": {
 22          "lr": 0.002,
 23          "keep": 8
 24        },
 25        "mean": 0.6527759283781052
 26      },
 27      {
 28        "cfg": {
 29          "lr": 0.002,
 30          "keep": 12
 31        },
 32        "mean": 0.6173136383295059
 33      },
 34      {
 35        "cfg": {
 36          "lr": 0.003,
 37          "keep": 4
 38        },
 39        "mean": 0.6469062119722366
 40      },
 41      {
 42        "cfg": {
 43          "lr": 0.003,
 44          "keep": 8
 45        },
 46        "mean": 0.6016789823770523
 47      },
 48      {
 49        "cfg": {
 50          "lr": 0.003,
 51          "keep": 12
 52        },
 53        "mean": 0.5432282239198685
 54      },
 55      {
 56        "cfg": {
 57          "lr": 0.004,
 58          "keep": 4
 59        },
 60        "mean": 0.5868831127882004
 61      },
 62      {
 63        "cfg": {
 64          "lr": 0.004,
 65          "keep": 8
 66        },
 67        "mean": 0.5453523471951485
 68      },
 69      {
 70        "cfg": {
 71          "lr": 0.004,
 72          "keep": 12
 73        },
 74        "mean": 0.5168638676404953
 75      }
 76    ],
 77    "full": {
 78      "mean": 0.5058855153620243,
 79      "std": 0.03455579185574975,
 80      "per_seed": [
 81        0.5703754425048828,
 82        0.48083919286727905,
 83        0.5038946270942688,
 84        0.5123462080955505,
 85        0.5410808324813843,
 86        0.5025615096092224,
 87        0.48530539870262146,
 88        0.4506809115409851
 89      ],
 90      "n": 8
 91    }
 92  },
 93  "idea": {
 94    "mean": 0.5126509107649326,
 95    "std": 0.03918740779455468,
 96    "per_seed": [
 97      0.5694426894187927,
 98      0.48401522636413574,
 99      0.5518521070480347,
100      0.500401496887207,
101      0.5515084862709045,
102      0.5079619288444519,
103      0.4907732903957367,
104      0.44525206089019775
105    ],
106    "n": 8
107  },
108  "comparison": {
109    "delta_mean": 0.006765395402908325,
110    "idea_wins": 3,
111    "n_pairs": 8,
112    "per_seed_diffs": [
113      -0.0009327530860900879,
114      0.0031760334968566895,
115      0.04795747995376587,
116      -0.011944711208343506,
117      0.010427653789520264,
118      0.005400419235229492,
119      0.005467891693115234,
120      -0.0054288506507873535
121    ],
122    "p_value": 0.3883,
123    "mde": 0.015105943689001283,
124    "mde_rel_pct": 2.9860399695751503,
125    "verdict": "no significant win",
126    "system_worked": false
127  },
128  "mechanism_signature": {
129    "math_check": {
130      "pca_residual": 1.910288129226306,
131      "rotation_residual": 2.8467599006415716,
132      "pca_le_rotation": true,
133      "topk_formula_error": 0.0
134    },
135    "trained_model_signature": {
136      "predicted_pca_retained_energy": 5.444458019758442,
137      "observed_topk_retained_fraction": 0.9996704459190369,
138      "mean_selection_gap": 0.10093563050031662,
139      "confirmed": true,
140      "source": "trained sequence model hidden states"
141    },
142    "idea_cfg": {
143      "lr": 0.004,
144      "keep": 12
145    },
146    "track_match": "multi-token sequence forecast; activation bottleneck is inserted in the transformer hidden sequence."
147  },
148  "idea_sweep": [
149    {
150      "cfg": {
151        "lr": 0.004,
152        "keep": 12
153      },
154      "result": {
155        "mean": 0.5126509107649326,
156        "std": 0.03918740779455468,
157        "per_seed": [
158          0.5694426894187927,
159          0.48401522636413574,
160          0.5518521070480347,
161          0.500401496887207,
162          0.5515084862709045,
163          0.5079619288444519,
164          0.4907732903957367,
165          0.44525206089019775
166        ],
167        "n": 8
168      }
169    },
170    {
171      "cfg": {
172        "lr": 0.002,
173        "keep": 12
174      },
175      "result": {
176        "mean": 0.5912451855838299,
177        "std": 0.05238796828411388,
178        "per_seed": [
179          0.6840536594390869,
180          0.5757109522819519,
181          0.5977234840393066,
182          0.5731315016746521,
183          0.6317427158355713,
184          0.6175632476806641,
185          0.553534746170044,
186          0.4965011775493622
187        ],
188        "n": 8
189      }
190    },
191    {
192      "cfg": {
193        "lr": 0.004,
194        "keep": 12
195      },
196      "result": {
197        "mean": 0.5126509107649326,
198        "std": 0.03918740779455468,
199        "per_seed": [
200          0.5694426894187927,
201          0.48401522636413574,
202          0.5518521070480347,
203          0.500401496887207,
204          0.5515084862709045,
205          0.5079619288444519,
206          0.4907732903957367,
207          0.44525206089019775
208        ],
209        "n": 8
210      }
211    }
212  ],
213  "device": "cuda"
214}