Exact doubly stochastic low-rank attention / results.json
Mechanism confirmed, baseline not beaten
1{
2 "prediction_1": {
3 "claim": "residuals are numerical-zero independent of n,r",
4 "sweep": [
5 {
6 "n": 16,
7 "r": 2,
8 "iterations": 4,
9 "max_row_residual": 1.1102230246251565e-16,
10 "max_shared_column_residual": 3.552713678800501e-15,
11 "projected_KL": 22.349774808788002,
12 "reference_row_normalized_KL": 21.970317896801408,
13 "hessian_null_norm": 4.782987168225453e-15
14 },
15 {
16 "n": 16,
17 "r": 4,
18 "iterations": 4,
19 "max_row_residual": 2.220446049250313e-16,
20 "max_shared_column_residual": 3.552713678800501e-15,
21 "projected_KL": 81.4472146488221,
22 "reference_row_normalized_KL": 81.23143408788488,
23 "hessian_null_norm": 1.7199501139797033e-15
24 },
25 {
26 "n": 16,
27 "r": 8,
28 "iterations": 4,
29 "max_row_residual": 2.220446049250313e-16,
30 "max_shared_column_residual": 7.105427357601002e-15,
31 "projected_KL": 270.3327018111314,
32 "reference_row_normalized_KL": 270.0767076299735,
33 "hessian_null_norm": 1.961044852356119e-15
34 },
35 {
36 "n": 16,
37 "r": 16,
38 "iterations": 4,
39 "max_row_residual": 2.220446049250313e-16,
40 "max_shared_column_residual": 7.405187574249794e-13,
41 "projected_KL": 639.6280424342364,
42 "reference_row_normalized_KL": 638.849357586517,
43 "hessian_null_norm": 9.930136612989092e-16
44 },
45 {
46 "n": 32,
47 "r": 2,
48 "iterations": 4,
49 "max_row_residual": 1.1102230246251565e-16,
50 "max_shared_column_residual": 5.329070518200751e-15,
51 "projected_KL": 76.88858258234636,
52 "reference_row_normalized_KL": 76.47797845540806,
53 "hessian_null_norm": 5.617333549722722e-15
54 },
55 {
56 "n": 32,
57 "r": 4,
58 "iterations": 4,
59 "max_row_residual": 2.220446049250313e-16,
60 "max_shared_column_residual": 1.7763568394002505e-15,
61 "projected_KL": 267.0935436685017,
62 "reference_row_normalized_KL": 267.0568060243071,
63 "hessian_null_norm": 2.175583928816829e-15
64 },
65 {
66 "n": 32,
67 "r": 8,
68 "iterations": 4,
69 "max_row_residual": 2.220446049250313e-16,
70 "max_shared_column_residual": 4.574118861455645e-13,
71 "projected_KL": 648.1912323747913,
72 "reference_row_normalized_KL": 647.3843260564644,
73 "hessian_null_norm": 2.945754565036118e-15
74 },
75 {
76 "n": 32,
77 "r": 16,
78 "iterations": 4,
79 "max_row_residual": 2.220446049250313e-16,
80 "max_shared_column_residual": 1.7763568394002505e-15,
81 "projected_KL": 1432.9330750053655,
82 "reference_row_normalized_KL": 1432.5761948821998,
83 "hessian_null_norm": 2.5023156146241385e-15
84 },
85 {
86 "n": 64,
87 "r": 2,
88 "iterations": 4,
89 "max_row_residual": 2.220446049250313e-16,
90 "max_shared_column_residual": 1.7763568394002505e-14,
91 "projected_KL": 200.50463638080583,
92 "reference_row_normalized_KL": 200.00809649415328,
93 "hessian_null_norm": 2.5618982671915014e-14
94 },
95 {
96 "n": 64,
97 "r": 4,
98 "iterations": 4,
99 "max_row_residual": 2.220446049250313e-16,
100 "max_shared_column_residual": 1.2434497875801753e-14,
101 "projected_KL": 459.5302941648396,
102 "reference_row_normalized_KL": 459.18127872487105,
103 "hessian_null_norm": 1.0509068785146553e-14
104 },
105 {
106 "n": 64,
107 "r": 8,
108 "iterations": 4,
109 "max_row_residual": 2.220446049250313e-16,
110 "max_shared_column_residual": 1.1546319456101628e-14,
111 "projected_KL": 1233.0106208627142,
112 "reference_row_normalized_KL": 1232.1658981278708,
113 "hessian_null_norm": 1.3440578383968888e-14
114 },
115 {
116 "n": 64,
117 "r": 16,
118 "iterations": 4,
119 "max_row_residual": 2.220446049250313e-16,
120 "max_shared_column_residual": 7.105427357601002e-15,
121 "projected_KL": 2791.8952543271416,
122 "reference_row_normalized_KL": 2791.3472049464085,
123 "hessian_null_norm": 6.993523870246153e-15
124 },
125 {
126 "n": 128,
127 "r": 2,
128 "iterations": 4,
129 "max_row_residual": 2.220446049250313e-16,
130 "max_shared_column_residual": 0.0,
131 "projected_KL": 294.05135467779064,
132 "reference_row_normalized_KL": 293.9327992195989,
133 "hessian_null_norm": 4.5496934130773415e-14
134 },
135 {
136 "n": 128,
137 "r": 4,
138 "iterations": 4,
139 "max_row_residual": 2.220446049250313e-16,
140 "max_shared_column_residual": 2.842170943040401e-14,
141 "projected_KL": 947.790297888946,
142 "reference_row_normalized_KL": 947.2156128648421,
143 "hessian_null_norm": 2.2748467065386707e-14
144 },
145 {
146 "n": 128,
147 "r": 8,
148 "iterations": 4,
149 "max_row_residual": 2.220446049250313e-16,
150 "max_shared_column_residual": 2.1316282072803006e-14,
151 "projected_KL": 2548.2812386598325,
152 "reference_row_normalized_KL": 2547.8086204853953,
153 "hessian_null_norm": 2.869792300935272e-14
154 },
155 {
156 "n": 128,
157 "r": 16,
158 "iterations": 4,
159 "max_row_residual": 2.220446049250313e-16,
160 "max_shared_column_residual": 7.105427357601002e-15,
161 "projected_KL": 5689.807386944296,
162 "reference_row_normalized_KL": 5689.261505681804,
163 "hessian_null_norm": 1.703821625409574e-14
164 },
165 {
166 "n": 256,
167 "r": 2,
168 "iterations": 3,
169 "max_row_residual": 2.220446049250313e-16,
170 "max_shared_column_residual": 2.984279490192421e-13,
171 "projected_KL": 690.3801290623901,
172 "reference_row_normalized_KL": 690.2604154191021,
173 "hessian_null_norm": 1.2224639458285405e-13
174 },
175 {
176 "n": 256,
177 "r": 4,
178 "iterations": 4,
179 "max_row_residual": 2.220446049250313e-16,
180 "max_shared_column_residual": 7.105427357601002e-14,
181 "projected_KL": 1957.818301241046,
182 "reference_row_normalized_KL": 1957.7563239924232,
183 "hessian_null_norm": 6.665481691674458e-14
184 },
185 {
186 "n": 256,
187 "r": 8,
188 "iterations": 4,
189 "max_row_residual": 2.220446049250313e-16,
190 "max_shared_column_residual": 4.973799150320701e-14,
191 "projected_KL": 4785.057366239143,
192 "reference_row_normalized_KL": 4784.625678593483,
193 "hessian_null_norm": 5.3290705182007514e-14
194 },
195 {
196 "n": 256,
197 "r": 16,
198 "iterations": 4,
199 "max_row_residual": 2.220446049250313e-16,
200 "max_shared_column_residual": 4.618527782440651e-14,
201 "projected_KL": 11319.745588977632,
202 "reference_row_normalized_KL": 11319.284375240084,
203 "hessian_null_norm": 3.940150075155706e-14
204 }
205 ]
206 },
207 "prediction_2": {
208 "claim": "covariance-sum HVP is exact and reduced curvature is positive",
209 "sweep": [
210 {
211 "r": 2,
212 "predicted_relative_error": 0.0,
213 "observed_relative_error": 1.1423217667413144e-13,
214 "predicted_positive_gauge_curvature": "> 0",
215 "observed_min_reduced_eigenvalue": 21.503171812151656
216 },
217 {
218 "r": 4,
219 "predicted_relative_error": 0.0,
220 "observed_relative_error": 2.0700800335795706e-16,
221 "predicted_positive_gauge_curvature": "> 0",
222 "observed_min_reduced_eigenvalue": 5.994879936143085
223 },
224 {
225 "r": 8,
226 "predicted_relative_error": 0.0,
227 "observed_relative_error": 3.9327182635266995e-16,
228 "predicted_positive_gauge_curvature": "> 0",
229 "observed_min_reduced_eigenvalue": 1.6945364246549077
230 },
231 {
232 "r": 12,
233 "predicted_relative_error": 0.0,
234 "observed_relative_error": 3.9198222699432094e-16,
235 "predicted_positive_gauge_curvature": "> 0",
236 "observed_min_reduced_eigenvalue": 0.7834050345856374
237 },
238 {
239 "r": 16,
240 "predicted_relative_error": 0.0,
241 "observed_relative_error": 2.9076393011614695e-16,
242 "predicted_positive_gauge_curvature": "> 0",
243 "observed_min_reduced_eigenvalue": 0.4840439077177008
244 }
245 ]
246 },
247 "prediction_3": {
248 "claim": "factor storage scales as n^2/(2nr+r), and thin equals dense factor application",
249 "sweep": [
250 {
251 "n": 32,
252 "r": 4,
253 "predicted_output_relative_error": 0.0,
254 "observed_output_relative_error": 2.936797370715141e-16,
255 "predicted_storage_ratio": 3.9384615384615387,
256 "observed_storage_ratio": 3.9384615384615387,
257 "dense_entries": 1024,
258 "factor_entries": 260
259 },
260 {
261 "n": 64,
262 "r": 4,
263 "predicted_output_relative_error": 0.0,
264 "observed_output_relative_error": 3.386865835672514e-16,
265 "predicted_storage_ratio": 7.937984496124031,
266 "observed_storage_ratio": 7.937984496124031,
267 "dense_entries": 4096,
268 "factor_entries": 516
269 },
270 {
271 "n": 128,
272 "r": 8,
273 "predicted_output_relative_error": 0.0,
274 "observed_output_relative_error": 4.951231020480598e-16,
275 "predicted_storage_ratio": 7.968871595330739,
276 "observed_storage_ratio": 7.968871595330739,
277 "dense_entries": 16384,
278 "factor_entries": 2056
279 },
280 {
281 "n": 256,
282 "r": 8,
283 "predicted_output_relative_error": 0.0,
284 "observed_output_relative_error": 6.776688632570342e-16,
285 "predicted_storage_ratio": 15.968810916179336,
286 "observed_storage_ratio": 15.968810916179336,
287 "dense_entries": 65536,
288 "factor_entries": 4104
289 },
290 {
291 "n": 512,
292 "r": 16,
293 "predicted_output_relative_error": 0.0,
294 "observed_output_relative_error": 6.422168098100112e-16,
295 "predicted_storage_ratio": 15.984390243902439,
296 "observed_storage_ratio": 15.984390243902439,
297 "dense_entries": 262144,
298 "factor_entries": 16400
299 }
300 ]
301 },
302 "mini_experiment": {
303 "n": 512,
304 "r": 16,
305 "d": 32,
306 "dense_apply_median_ms": 1.2821340060327202,
307 "thin_apply_median_ms": 0.24867449974408373,
308 "speedup": 5.15587246521936,
309 "projection_shared_column_residual": 7.460698725481052e-14,
310 "dense_attention_state_entries": 262144,
311 "idea_factor_entries": 16400
312 }
313}