1{
  2  "bench_version": 1,
  3  "track": "sequence",
  4  "model": "transformer_tiny",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "best_cfg": {
  9      "lr": 0.001,
 10      "epochs": 10
 11    },
 12    "sweep": [
 13      {
 14        "cfg": {
 15          "lr": 0.001,
 16          "epochs": 10
 17        },
 18        "mean": 0.482548501342535
 19      },
 20      {
 21        "cfg": {
 22          "lr": 0.003,
 23          "epochs": 10
 24        },
 25        "mean": 0.6393650658428669
 26      },
 27      {
 28        "cfg": {
 29          "lr": 0.01,
 30          "epochs": 10
 31        },
 32        "mean": 1.4253769740462303
 33      }
 34    ],
 35    "full": {
 36      "mean": 0.482548501342535,
 37      "std": 0.03354559876849172,
 38      "per_seed": [
 39        0.4842072129249573,
 40        0.42341384291648865,
 41        0.4566255807876587,
 42        0.5167749524116516,
 43        0.49487417936325073,
 44        0.4824545681476593,
 45        0.5386028289794922,
 46        0.4634348452091217
 47      ],
 48      "n": 8
 49    }
 50  },
 51  "idea": {
 52    "best_cfg": {
 53      "lr": 0.001,
 54      "epochs": 10
 55    },
 56    "sweep": [
 57      {
 58        "cfg": {
 59          "lr": 0.001,
 60          "epochs": 10
 61        },
 62        "mean": 0.4744987264275551
 63      },
 64      {
 65        "cfg": {
 66          "lr": 0.003,
 67          "epochs": 10
 68        },
 69        "mean": 0.5909863151609898
 70      },
 71      {
 72        "cfg": {
 73          "lr": 0.01,
 74          "epochs": 10
 75        },
 76        "mean": 1.119819551706314
 77      }
 78    ],
 79    "mean": 0.4744987264275551,
 80    "std": 0.052674658740946755,
 81    "per_seed": [
 82      0.49082696437835693,
 83      0.3496098220348358,
 84      0.49417421221733093,
 85      0.4901917278766632,
 86      0.519122302532196,
 87      0.44667643308639526,
 88      0.5286210179328918,
 89      0.47676733136177063
 90    ],
 91    "n": 8
 92  },
 93  "comparison": {
 94    "delta_mean": -0.008049774914979935,
 95    "idea_wins": 4,
 96    "n_pairs": 8,
 97    "per_seed_diffs": [
 98      0.006619751453399658,
 99      -0.07380402088165283,
100      0.03754863142967224,
101      -0.026583224534988403,
102      0.024248123168945312,
103      -0.03577813506126404,
104      -0.009981811046600342,
105      0.013332486152648926
106    ],
107    "p_value": 0.57185,
108    "mde": 0.030369881877730317,
109    "mde_rel_pct": 6.293643394028983,
110    "verdict": "no significant win",
111    "system_worked": false
112  },
113  "track_match": "sequence has multi-token correlations and transformer attention; therefore it is structurally matched",
114  "attention_audit": {
115    "masked_edges": 94,
116    "dense_edges": 1024,
117    "per_seed": [
118      {
119        "seed": 0,
120        "test_mse": 0.49082696437835693,
121        "edges": 94,
122        "n_tokens": 32,
123        "max_shared_keys": 1
124      },
125      {
126        "seed": 1,
127        "test_mse": 0.3496098220348358,
128        "edges": 94,
129        "n_tokens": 32,
130        "max_shared_keys": 1
131      },
132      {
133        "seed": 2,
134        "test_mse": 0.49417421221733093,
135        "edges": 94,
136        "n_tokens": 32,
137        "max_shared_keys": 1
138      },
139      {
140        "seed": 3,
141        "test_mse": 0.4901917278766632,
142        "edges": 94,
143        "n_tokens": 32,
144        "max_shared_keys": 1
145      },
146      {
147        "seed": 4,
148        "test_mse": 0.519122302532196,
149        "edges": 94,
150        "n_tokens": 32,
151        "max_shared_keys": 1
152      },
153      {
154        "seed": 5,
155        "test_mse": 0.44667643308639526,
156        "edges": 94,
157        "n_tokens": 32,
158        "max_shared_keys": 1
159      },
160      {
161        "seed": 6,
162        "test_mse": 0.5286210179328918,
163        "edges": 94,
164        "n_tokens": 32,
165        "max_shared_keys": 1
166      },
167      {
168        "seed": 7,
169        "test_mse": 0.47676733136177063,
170        "edges": 94,
171        "n_tokens": 32,
172        "max_shared_keys": 1
173      }
174    ]
175  },
176  "mechanism_signature": {
177    "prediction": "hierarchical repaired attention has linear active-edge count and K2,2 maximum common-neighbor count <=1",
178    "observed_edges": 94,
179    "observed_dense_edges": 1024,
180    "edge_reduction": 10.893617021276595,
181    "observed_edge_scaling_exponent": 1.0,
182    "observed_max_shared_keys": 1,
183    "trained_model_test_mse_mean": 0.4744987264275551,
184    "confirmed": true
185  }
186}