1{
  2  "bench_version": 1,
  3  "track": "sequence",
  4  "model": "transformer_tiny",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "best_cfg": {
  9      "lr": 0.001,
 10      "rank": 8
 11    },
 12    "sweep": [
 13      {
 14        "cfg": {
 15          "lr": 0.001,
 16          "rank": 8
 17        },
 18        "mean": 0.4039061442017555
 19      },
 20      {
 21        "cfg": {
 22          "lr": 0.003,
 23          "rank": 8
 24        },
 25        "mean": 0.5307872667908669
 26      },
 27      {
 28        "cfg": {
 29          "lr": 0.006,
 30          "rank": 8
 31        },
 32        "mean": 0.8058461993932724
 33      }
 34    ],
 35    "full": {
 36      "mean": 0.4480689875781536,
 37      "std": 0.08669399607049459,
 38      "per_seed": [
 39        0.4437996447086334,
 40        0.3383534550666809,
 41        0.37107908725738525,
 42        0.4623923897743225,
 43        0.6176453828811646,
 44        0.3899054229259491,
 45        0.5426207780838013,
 46        0.41875573992729187
 47      ],
 48      "n": 8
 49    },
 50    "selected_cfg_full": {
 51      "mean": 0.4480689875781536,
 52      "std": 0.08669399607049459,
 53      "per_seed": [
 54        0.4437996447086334,
 55        0.3383534550666809,
 56        0.37107908725738525,
 57        0.4623923897743225,
 58        0.6176453828811646,
 59        0.3899054229259491,
 60        0.5426207780838013,
 61        0.41875573992729187
 62      ],
 63      "n": 8
 64    }
 65  },
 66  "idea": {
 67    "mean": 0.4980953820049763,
 68    "std": 0.0717662616842552,
 69    "per_seed": [
 70      0.5194478631019592,
 71      0.45840978622436523,
 72      0.43587931990623474,
 73      0.6198630928993225,
 74      0.5728811025619507,
 75      0.39179709553718567,
 76      0.535282552242279,
 77      0.45120224356651306
 78    ],
 79    "n": 8
 80  },
 81  "comparison": {
 82    "delta_mean": 0.05002639442682266,
 83    "idea_wins": 2,
 84    "n_pairs": 8,
 85    "per_seed_diffs": [
 86      0.0756482183933258,
 87      0.12005633115768433,
 88      0.06480023264884949,
 89      0.157470703125,
 90      -0.04476428031921387,
 91      0.0018916726112365723,
 92      -0.007338225841522217,
 93      0.03244650363922119
 94    ],
 95    "p_value": 0.08665,
 96    "mde": 0.05673288695634186,
 97    "mde_rel_pct": 12.661641070717113,
 98    "verdict": "no significant win",
 99    "system_worked": false
100  },
101  "mechanism_signature": {
102    "prediction": "trained low-rank attention remains row-stochastic and shared-marginal, with O(nr) state",
103    "observed": {
104      "row_u": 2.086162567138672e-07,
105      "row_v": 2.2351741790771484e-07,
106      "shared_col": 2.5033950805664062e-06,
107      "factor_entries": 520,
108      "dense_entries": 1024,
109      "predicted_storage_ratio": 1.9692307692307693,
110      "observed_storage_ratio": 1.9692307692307693,
111      "confirmed": true
112    },
113    "confirmed": true
114  },
115  "protocol_notes": {
116    "matched_track": "sequence multi-token forecast",
117    "shared_architecture": "same input/position/FFN/head; only attention operator differs",
118    "n_train": 400,
119    "epochs": 8,
120    "idea_grid": [
121      [
122        {
123          "mean": 0.4980953820049763,
124          "std": 0.0717662616842552,
125          "per_seed": [
126            0.5194478631019592,
127            0.45840978622436523,
128            0.43587931990623474,
129            0.6198630928993225,
130            0.5728811025619507,
131            0.39179709553718567,
132            0.535282552242279,
133            0.45120224356651306
134          ],
135          "n": 8
136        },
137        {
138          "lr": 0.001,
139          "rank": 8
140        }
141      ],
142      [
143        {
144          "mean": 0.6111128553748131,
145          "std": 0.07747015035751928,
146          "per_seed": [
147            0.6425154209136963,
148            0.5512837767601013,
149            0.5704087018966675,
150            0.7150149941444397,
151            0.646016538143158,
152            0.5375900864601135,
153            0.7234078645706177,
154            0.5026654601097107
155          ],
156          "n": 8
157        },
158        {
159          "lr": 0.003,
160          "rank": 8
161        }
162      ],
163      [
164        {
165          "mean": 0.9795129522681236,
166          "std": 0.22563627392238808,
167          "per_seed": [
168            0.9576879739761353,
169            0.7885317802429199,
170            1.4408782720565796,
171            1.1196062564849854,
172            0.7925991415977478,
173            0.8517841100692749,
174            1.1472877264022827,
175            0.7377283573150635
176          ],
177          "n": 8
178        },
179        {
180          "lr": 0.006,
181          "rank": 8
182        }
183      ]
184    ]
185  }
186}