Rank-One Delta Associative Memory / bench_report.json
Mechanism confirmed, baseline not beaten
1{
2 "bench_version": 1,
3 "track": "sequence",
4 "model": "transformer_tiny",
5 "metric_direction": "lower is better",
6 "n_seeds": 8,
7 "baseline": {
8 "best_cfg": {
9 "lr": 0.0015
10 },
11 "sweep": [
12 {
13 "cfg": {
14 "lr": 0.0015
15 },
16 "mean": 0.987314760684967
17 },
18 {
19 "cfg": {
20 "lr": 0.003
21 },
22 "mean": 1.5178270637989044
23 },
24 {
25 "cfg": {
26 "lr": 0.006
27 },
28 "mean": 2.574708193540573
29 }
30 ],
31 "full": {
32 "mean": 1.0789237320423126,
33 "std": 0.1999035176851697,
34 "per_seed": [
35 0.6216103434562683,
36 1.084313154220581,
37 0.9683473706245422,
38 1.2749881744384766,
39 1.0933938026428223,
40 1.1413570642471313,
41 1.1405138969421387,
42 1.3068660497665405
43 ],
44 "n": 8
45 }
46 },
47 "idea": {
48 "mean": 1.1184165999293327,
49 "std": 0.17711685282328676,
50 "per_seed": [
51 1.2109392881393433,
52 0.8607760071754456,
53 0.8473414182662964,
54 1.2753307819366455,
55 1.0955066680908203,
56 1.1688036918640137,
57 1.0974258184432983,
58 1.3912091255187988
59 ],
60 "n": 8,
61 "cfg": {
62 "lr": 0.0015,
63 "beta": 0.5
64 },
65 "mechanism_signatures": [
66 {
67 "state_norm": 0.11540921777486801,
68 "update_norm": 0.6236135959625244,
69 "read_norm": 0.5206019282341003
70 },
71 {
72 "state_norm": 0.1203140988945961,
73 "update_norm": 0.7720737457275391,
74 "read_norm": 0.5169996619224548
75 },
76 {
77 "state_norm": 0.11553805321455002,
78 "update_norm": 0.6452788710594177,
79 "read_norm": 0.5866501927375793
80 },
81 {
82 "state_norm": 0.1047215536236763,
83 "update_norm": 0.48125800490379333,
84 "read_norm": 0.563747763633728
85 },
86 {
87 "state_norm": 0.10959173738956451,
88 "update_norm": 0.8530905246734619,
89 "read_norm": 0.4561031460762024
90 },
91 {
92 "state_norm": 0.10967692732810974,
93 "update_norm": 0.5639088153839111,
94 "read_norm": 0.5506287217140198
95 },
96 {
97 "state_norm": 0.10660543292760849,
98 "update_norm": 0.5371220111846924,
99 "read_norm": 0.5531039237976074
100 },
101 {
102 "state_norm": 0.11331523954868317,
103 "update_norm": 0.6734405755996704,
104 "read_norm": 0.5476616621017456
105 }
106 ]
107 },
108 "comparison": {
109 "delta_mean": 0.03949286788702011,
110 "idea_wins": 3,
111 "n_pairs": 8,
112 "per_seed_diffs": [
113 0.589328944683075,
114 -0.2235371470451355,
115 -0.12100595235824585,
116 0.0003426074981689453,
117 0.002112865447998047,
118 0.027446627616882324,
119 -0.04308807849884033,
120 0.0843430757522583
121 ],
122 "p_value": 0.82115,
123 "mde": 0.20215727076091933,
124 "mde_rel_pct": 18.73693800193388,
125 "verdict": "no significant win",
126 "system_worked": false
127 },
128 "mechanism_signature": {
129 "prediction": "rank-one per-sample state has finite norm and nonzero update/read activity",
130 "observed": [
131 {
132 "state_norm": 0.11540921777486801,
133 "update_norm": 0.6236135959625244,
134 "read_norm": 0.5206019282341003
135 },
136 {
137 "state_norm": 0.1203140988945961,
138 "update_norm": 0.7720737457275391,
139 "read_norm": 0.5169996619224548
140 },
141 {
142 "state_norm": 0.11553805321455002,
143 "update_norm": 0.6452788710594177,
144 "read_norm": 0.5866501927375793
145 },
146 {
147 "state_norm": 0.1047215536236763,
148 "update_norm": 0.48125800490379333,
149 "read_norm": 0.563747763633728
150 },
151 {
152 "state_norm": 0.10959173738956451,
153 "update_norm": 0.8530905246734619,
154 "read_norm": 0.4561031460762024
155 },
156 {
157 "state_norm": 0.10967692732810974,
158 "update_norm": 0.5639088153839111,
159 "read_norm": 0.5506287217140198
160 },
161 {
162 "state_norm": 0.10660543292760849,
163 "update_norm": 0.5371220111846924,
164 "read_norm": 0.5531039237976074
165 },
166 {
167 "state_norm": 0.11331523954868317,
168 "update_norm": 0.6734405755996704,
169 "read_norm": 0.5476616621017456
170 }
171 ],
172 "idea_sweep": [
173 {
174 "mean": 1.1184165999293327,
175 "std": 0.17711685282328676,
176 "per_seed": [
177 1.2109392881393433,
178 0.8607760071754456,
179 0.8473414182662964,
180 1.2753307819366455,
181 1.0955066680908203,
182 1.1688036918640137,
183 1.0974258184432983,
184 1.3912091255187988
185 ],
186 "n": 8,
187 "cfg": {
188 "lr": 0.0015,
189 "beta": 0.5
190 },
191 "mechanism_signatures": [
192 {
193 "state_norm": 0.11540921777486801,
194 "update_norm": 0.6236135959625244,
195 "read_norm": 0.5206019282341003
196 },
197 {
198 "state_norm": 0.1203140988945961,
199 "update_norm": 0.7720737457275391,
200 "read_norm": 0.5169996619224548
201 },
202 {
203 "state_norm": 0.11553805321455002,
204 "update_norm": 0.6452788710594177,
205 "read_norm": 0.5866501927375793
206 },
207 {
208 "state_norm": 0.1047215536236763,
209 "update_norm": 0.48125800490379333,
210 "read_norm": 0.563747763633728
211 },
212 {
213 "state_norm": 0.10959173738956451,
214 "update_norm": 0.8530905246734619,
215 "read_norm": 0.4561031460762024
216 },
217 {
218 "state_norm": 0.10967692732810974,
219 "update_norm": 0.5639088153839111,
220 "read_norm": 0.5506287217140198
221 },
222 {
223 "state_norm": 0.10660543292760849,
224 "update_norm": 0.5371220111846924,
225 "read_norm": 0.5531039237976074
226 },
227 {
228 "state_norm": 0.11331523954868317,
229 "update_norm": 0.6734405755996704,
230 "read_norm": 0.5476616621017456
231 }
232 ]
233 },
234 {
235 "mean": 1.396845631301403,
236 "std": 0.3391511114620585,
237 "per_seed": [
238 0.9106339812278748,
239 1.3027052879333496,
240 1.3874930143356323,
241 0.8688368797302246,
242 1.8346285820007324,
243 1.7065383195877075,
244 1.4406466484069824,
245 1.7232823371887207
246 ],
247 "n": 8,
248 "cfg": {
249 "lr": 0.003,
250 "beta": 0.5
251 },
252 "mechanism_signatures": [
253 {
254 "state_norm": 0.12106544524431229,
255 "update_norm": 0.7507100105285645,
256 "read_norm": 0.5045618414878845
257 },
258 {
259 "state_norm": 0.10777172446250916,
260 "update_norm": 0.5398299694061279,
261 "read_norm": 0.5512951016426086
262 },
263 {
264 "state_norm": 0.13707156479358673,
265 "update_norm": 0.9250611662864685,
266 "read_norm": 0.526276707649231
267 },
268 {
269 "state_norm": 0.12658239901065826,
270 "update_norm": 0.7286500334739685,
271 "read_norm": 0.551619827747345
272 },
273 {
274 "state_norm": 0.11532067507505417,
275 "update_norm": 0.764352560043335,
276 "read_norm": 0.5137579441070557
277 },
278 {
279 "state_norm": 0.11622980982065201,
280 "update_norm": 0.6543875932693481,
281 "read_norm": 0.5391327142715454
282 },
283 {
284 "state_norm": 0.11485152691602707,
285 "update_norm": 0.6875792741775513,
286 "read_norm": 0.49508407711982727
287 },
288 {
289 "state_norm": 0.09085177630186081,
290 "update_norm": 0.47813138365745544,
291 "read_norm": 0.5154584646224976
292 }
293 ]
294 },
295 {
296 "mean": 1.7542085275053978,
297 "std": 1.03264163220941,
298 "per_seed": [
299 1.2069628238677979,
300 1.1996299028396606,
301 1.1370363235473633,
302 0.9978018403053284,
303 0.8372374773025513,
304 2.680882692337036,
305 1.9308146238327026,
306 4.043302536010742
307 ],
308 "n": 8,
309 "cfg": {
310 "lr": 0.006,
311 "beta": 0.5
312 },
313 "mechanism_signatures": [
314 {
315 "state_norm": 0.10249067097902298,
316 "update_norm": 0.49012985825538635,
317 "read_norm": 0.5499802827835083
318 },
319 {
320 "state_norm": 0.10208044946193695,
321 "update_norm": 0.32191911339759827,
322 "read_norm": 0.7767879366874695
323 },
324 {
325 "state_norm": 0.12325321137905121,
326 "update_norm": 0.5648367404937744,
327 "read_norm": 0.6759800910949707
328 },
329 {
330 "state_norm": 0.12312870472669601,
331 "update_norm": 0.4876507818698883,
332 "read_norm": 0.6405466794967651
333 },
334 {
335 "state_norm": 0.10362966358661652,
336 "update_norm": 0.5586110353469849,
337 "read_norm": 0.560613751411438
338 },
339 {
340 "state_norm": 0.11490625888109207,
341 "update_norm": 0.3960989713668823,
342 "read_norm": 0.5643085241317749
343 },
344 {
345 "state_norm": 0.10144433379173279,
346 "update_norm": 0.49015238881111145,
347 "read_norm": 0.6192492246627808
348 },
349 {
350 "state_norm": 0.07356467097997665,
351 "update_norm": 0.2797723710536957,
352 "read_norm": 0.5559394359588623
353 }
354 ]
355 }
356 ],
357 "parameter_counts": {
358 "baseline": 71169,
359 "idea": 79490
360 },
361 "confirmed": true
362 },
363 "protocol_note": "Official bench sequence track; baseline and idea share transformer encoder/head and paired datasets."
364}