1{
  2  "bench_version": 1,
  3  "track": "sequence",
  4  "model": "transformer_tiny",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "best_cfg": {
  9      "lr": 0.001,
 10      "tau": 2.0
 11    },
 12    "sweep": [
 13      {
 14        "cfg": {
 15          "lr": 0.001,
 16          "tau": 0.5
 17        },
 18        "mean": 0.5767685323953629
 19      },
 20      {
 21        "cfg": {
 22          "lr": 0.001,
 23          "tau": 1.0
 24        },
 25        "mean": 0.5731862112879753
 26      },
 27      {
 28        "cfg": {
 29          "lr": 0.001,
 30          "tau": 2.0
 31        },
 32        "mean": 0.5661996975541115
 33      },
 34      {
 35        "cfg": {
 36          "lr": 0.003,
 37          "tau": 0.5
 38        },
 39        "mean": 0.7547104060649872
 40      },
 41      {
 42        "cfg": {
 43          "lr": 0.003,
 44          "tau": 1.0
 45        },
 46        "mean": 0.6658876091241837
 47      },
 48      {
 49        "cfg": {
 50          "lr": 0.003,
 51          "tau": 2.0
 52        },
 53        "mean": 0.7227341532707214
 54      },
 55      {
 56        "cfg": {
 57          "lr": 0.01,
 58          "tau": 0.5
 59        },
 60        "mean": 1.189420223236084
 61      },
 62      {
 63        "cfg": {
 64          "lr": 0.01,
 65          "tau": 1.0
 66        },
 67        "mean": 1.965364009141922
 68      },
 69      {
 70        "cfg": {
 71          "lr": 0.01,
 72          "tau": 2.0
 73        },
 74        "mean": 1.9628772139549255
 75      }
 76    ],
 77    "full": {
 78      "mean": 0.5629024468362331,
 79      "std": 0.0610525769421287,
 80      "per_seed": [
 81        0.639130711555481,
 82        0.47835078835487366,
 83        0.5805663466453552,
 84        0.5667509436607361,
 85        0.6444448828697205,
 86        0.4818141460418701,
 87        0.5961759686470032,
 88        0.5159857869148254
 89      ],
 90      "n": 8
 91    }
 92  },
 93  "idea": {
 94    "mean": 0.5631052926182747,
 95    "std": 0.05872795158375719,
 96    "per_seed": [
 97      0.6507832407951355,
 98      0.4800701141357422,
 99      0.6299005746841431,
100      0.5685753226280212,
101      0.5922347903251648,
102      0.5042065382003784,
103      0.5788382887840271,
104      0.5002334713935852
105    ],
106    "n": 8
107  },
108  "comparison": {
109    "delta_mean": 0.00020284578204154968,
110    "idea_wins": 3,
111    "n_pairs": 8,
112    "per_seed_diffs": [
113      0.011652529239654541,
114      0.0017193257808685303,
115      0.04933422803878784,
116      0.0018243789672851562,
117      -0.052210092544555664,
118      0.0223923921585083,
119      -0.017337679862976074,
120      -0.015752315521240234
121    ],
122    "p_value": 0.9927,
123    "mde": 0.025187222915370296,
124    "mde_rel_pct": 4.47452716841682,
125    "verdict": "no measurable effect",
126    "system_worked": false
127  },
128  "mechanism_signature": {
129    "prediction": "role-selective exact queries should be more concentrated than mismatched queries",
130    "predicted_exact_minus_mismatched_mass": 0.2,
131    "observed_exact_role_mass": 0.03502586856484413,
132    "observed_mismatched_role_mass": 0.03239620546810329,
133    "observed_gap": 0.002629663096740842,
134    "confirmed": false,
135    "note": "probability mass is measured from attention of trained role-filler models on held-out sequence windows"
136  },
137  "idea_sweep": [
138    {
139      "cfg": {
140        "lr": 0.001,
141        "tau": 2.0
142      },
143      "result": {
144        "mean": 0.5641730912029743,
145        "std": 0.0628611757188025,
146        "per_seed": [
147          0.6497918367385864,
148          0.4751982092857361,
149          0.6231837868690491,
150          0.5758178234100342,
151          0.5922346711158752,
152          0.49082884192466736,
153          0.6094589233398438,
154          0.49687063694000244
155        ],
156        "n": 8
157      }
158    },
159    {
160      "cfg": {
161        "lr": 0.001,
162        "tau": 1.0
163      },
164      "result": {
165        "mean": 0.5631052926182747,
166        "std": 0.05872795158375719,
167        "per_seed": [
168          0.6507832407951355,
169          0.4800701141357422,
170          0.6299005746841431,
171          0.5685753226280212,
172          0.5922347903251648,
173          0.5042065382003784,
174          0.5788382887840271,
175          0.5002334713935852
176        ],
177        "n": 8
178      }
179    },
180    {
181      "cfg": {
182        "lr": 0.01,
183        "tau": 1.0
184      },
185      "result": {
186        "mean": 1.587875448167324,
187        "std": 0.6160067425650491,
188        "per_seed": [
189          1.209954857826233,
190          2.5395073890686035,
191          2.092766761779785,
192          0.9216881990432739,
193          0.9916467070579529,
194          1.3980218172073364,
195          2.4178340435028076,
196          1.1315838098526
197        ],
198        "n": 8
199      }
200    }
201  ]
202}