Sublinear-expander sparse attention / results.json
Unverified
1{
2 "epsilon_max_expander": 2.413897921625164,
3 "epsilon_max_ring": 2.413897921625164,
4 "epsilon_used": 1.9311183373001315,
5 "exact_expansion_rows": [
6 [
7 2,
8 2.0,
9 3.3333333333333335,
10 4.0,
11 3.2
12 ],
13 [
14 3,
15 1.3333333333333333,
16 2.761904761904762,
17 4.0,
18 2.5608832115299953
19 ],
20 [
21 4,
22 1.0,
23 2.2747252747252746,
24 3.0,
25 2.406076671975439
26 ],
27 [
28 5,
29 0.8,
30 1.8615384615384618,
31 2.2,
32 2.378321688323566
33 ],
34 [
35 6,
36 0.6666666666666666,
37 1.512820512820513,
38 1.6666666666666667,
39 2.4000000000000004
40 ],
41 [
42 7,
43 0.5714285714285714,
44 1.2197802197802199,
45 1.2857142857142858,
46 2.444910918074699
47 ],
48 [
49 8,
50 0.5,
51 0.9743589743589743,
52 1.0,
53 2.5019505294028463
54 ]
55 ],
56 "max_bound_minus_observed": 0.0,
57 "epsilon_scaling": [
58 {
59 "fraction_of_max": 0.25,
60 "epsilon": 0.603474480406291,
61 "max_violation": 0.0
62 },
63 {
64 "fraction_of_max": 0.5,
65 "epsilon": 1.206948960812582,
66 "max_violation": 0.0
67 },
68 {
69 "fraction_of_max": 0.75,
70 "epsilon": 1.810423441218873,
71 "max_violation": 0.0
72 },
73 {
74 "fraction_of_max": 1.0,
75 "epsilon": 2.413897921625164,
76 "max_violation": 0.0
77 },
78 {
79 "fraction_of_max": 1.1,
80 "epsilon": 2.655287713787681,
81 "max_violation": 0.20000000000000018
82 }
83 ],
84 "graph_family": [
85 {
86 "graph": "ring",
87 "epsilon_max": 2.413897921625164,
88 "sampled_rows": [
89 [
90 2,
91 2.0,
92 3.3,
93 1.6
94 ],
95 [
96 4,
97 1.0,
98 2.2420833333333334,
99 0.6015191679938597
100 ],
101 [
102 8,
103 0.625,
104 0.9733333333333334,
105 0.3127438161753558
106 ],
107 [
108 8,
109 0.5,
110 0.9733333333333334,
111 0.3127438161753558
112 ]
113 ]
114 },
115 {
116 "graph": "random_regular",
117 "epsilon_max": 2.413897921625164,
118 "sampled_rows": [
119 [
120 2,
121 2.0,
122 3.370833333333333,
123 1.6
124 ],
125 [
126 4,
127 1.0,
128 2.29375,
129 0.6015191679938597
130 ],
131 [
132 8,
133 0.625,
134 0.9752083333333333,
135 0.3127438161753558
136 ],
137 [
138 8,
139 0.5,
140 0.9752083333333333,
141 0.3127438161753558
142 ]
143 ]
144 }
145 ],
146 "growth_sweep": [
147 {
148 "d": 2,
149 "epsilon_max": 1.206948960812582,
150 "sweep": [
151 {
152 "fraction_of_max": 0.25,
153 "predicted_layers_to_32": 63,
154 "observed_layers_to_32": 9,
155 "reached": 33
156 },
157 {
158 "fraction_of_max": 0.5,
159 "predicted_layers_to_32": 32,
160 "observed_layers_to_32": 9,
161 "reached": 33
162 },
163 {
164 "fraction_of_max": 0.75,
165 "predicted_layers_to_32": 21,
166 "observed_layers_to_32": 9,
167 "reached": 33
168 },
169 {
170 "fraction_of_max": 1.0,
171 "predicted_layers_to_32": 16,
172 "observed_layers_to_32": 9,
173 "reached": 33
174 }
175 ]
176 },
177 {
178 "d": 4,
179 "epsilon_max": 2.413897921625164,
180 "sweep": [
181 {
182 "fraction_of_max": 0.25,
183 "predicted_layers_to_32": 32,
184 "observed_layers_to_32": 5,
185 "reached": 32
186 },
187 {
188 "fraction_of_max": 0.5,
189 "predicted_layers_to_32": 16,
190 "observed_layers_to_32": 5,
191 "reached": 32
192 },
193 {
194 "fraction_of_max": 0.75,
195 "predicted_layers_to_32": 11,
196 "observed_layers_to_32": 5,
197 "reached": 32
198 },
199 {
200 "fraction_of_max": 1.0,
201 "predicted_layers_to_32": 8,
202 "observed_layers_to_32": 5,
203 "reached": 32
204 }
205 ]
206 },
207 {
208 "d": 6,
209 "epsilon_max": 3.620846882437746,
210 "sweep": [
211 {
212 "fraction_of_max": 0.25,
213 "predicted_layers_to_32": 21,
214 "observed_layers_to_32": 4,
215 "reached": 35
216 },
217 {
218 "fraction_of_max": 0.5,
219 "predicted_layers_to_32": 11,
220 "observed_layers_to_32": 4,
221 "reached": 35
222 },
223 {
224 "fraction_of_max": 0.75,
225 "predicted_layers_to_32": 7,
226 "observed_layers_to_32": 4,
227 "reached": 35
228 },
229 {
230 "fraction_of_max": 1.0,
231 "predicted_layers_to_32": 6,
232 "observed_layers_to_32": 4,
233 "reached": 35
234 }
235 ]
236 },
237 {
238 "d": 8,
239 "epsilon_max": 4.827795843250328,
240 "sweep": [
241 {
242 "fraction_of_max": 0.25,
243 "predicted_layers_to_32": 16,
244 "observed_layers_to_32": 2,
245 "reached": 34
246 },
247 {
248 "fraction_of_max": 0.5,
249 "predicted_layers_to_32": 8,
250 "observed_layers_to_32": 2,
251 "reached": 34
252 },
253 {
254 "fraction_of_max": 0.75,
255 "predicted_layers_to_32": 6,
256 "observed_layers_to_32": 2,
257 "reached": 34
258 },
259 {
260 "fraction_of_max": 1.0,
261 "predicted_layers_to_32": 4,
262 "observed_layers_to_32": 2,
263 "reached": 34
264 }
265 ]
266 }
267 ],
268 "attention_ms": {
269 "dense": 0.3613884999746612,
270 "local": 1.3365088998398278,
271 "expander": 2.6804862999900556
272 },
273 "device": "cuda"
274}