Sublinear-expander sparse attention / results.json

Unverified

Raw ⬇ ZIP
  1{
  2  "epsilon_max_expander": 2.413897921625164,
  3  "epsilon_max_ring": 2.413897921625164,
  4  "epsilon_used": 1.9311183373001315,
  5  "exact_expansion_rows": [
  6    [
  7      2,
  8      2.0,
  9      3.3333333333333335,
 10      4.0,
 11      3.2
 12    ],
 13    [
 14      3,
 15      1.3333333333333333,
 16      2.761904761904762,
 17      4.0,
 18      2.5608832115299953
 19    ],
 20    [
 21      4,
 22      1.0,
 23      2.2747252747252746,
 24      3.0,
 25      2.406076671975439
 26    ],
 27    [
 28      5,
 29      0.8,
 30      1.8615384615384618,
 31      2.2,
 32      2.378321688323566
 33    ],
 34    [
 35      6,
 36      0.6666666666666666,
 37      1.512820512820513,
 38      1.6666666666666667,
 39      2.4000000000000004
 40    ],
 41    [
 42      7,
 43      0.5714285714285714,
 44      1.2197802197802199,
 45      1.2857142857142858,
 46      2.444910918074699
 47    ],
 48    [
 49      8,
 50      0.5,
 51      0.9743589743589743,
 52      1.0,
 53      2.5019505294028463
 54    ]
 55  ],
 56  "max_bound_minus_observed": 0.0,
 57  "epsilon_scaling": [
 58    {
 59      "fraction_of_max": 0.25,
 60      "epsilon": 0.603474480406291,
 61      "max_violation": 0.0
 62    },
 63    {
 64      "fraction_of_max": 0.5,
 65      "epsilon": 1.206948960812582,
 66      "max_violation": 0.0
 67    },
 68    {
 69      "fraction_of_max": 0.75,
 70      "epsilon": 1.810423441218873,
 71      "max_violation": 0.0
 72    },
 73    {
 74      "fraction_of_max": 1.0,
 75      "epsilon": 2.413897921625164,
 76      "max_violation": 0.0
 77    },
 78    {
 79      "fraction_of_max": 1.1,
 80      "epsilon": 2.655287713787681,
 81      "max_violation": 0.20000000000000018
 82    }
 83  ],
 84  "graph_family": [
 85    {
 86      "graph": "ring",
 87      "epsilon_max": 2.413897921625164,
 88      "sampled_rows": [
 89        [
 90          2,
 91          2.0,
 92          3.3,
 93          1.6
 94        ],
 95        [
 96          4,
 97          1.0,
 98          2.2420833333333334,
 99          0.6015191679938597
100        ],
101        [
102          8,
103          0.625,
104          0.9733333333333334,
105          0.3127438161753558
106        ],
107        [
108          8,
109          0.5,
110          0.9733333333333334,
111          0.3127438161753558
112        ]
113      ]
114    },
115    {
116      "graph": "random_regular",
117      "epsilon_max": 2.413897921625164,
118      "sampled_rows": [
119        [
120          2,
121          2.0,
122          3.370833333333333,
123          1.6
124        ],
125        [
126          4,
127          1.0,
128          2.29375,
129          0.6015191679938597
130        ],
131        [
132          8,
133          0.625,
134          0.9752083333333333,
135          0.3127438161753558
136        ],
137        [
138          8,
139          0.5,
140          0.9752083333333333,
141          0.3127438161753558
142        ]
143      ]
144    }
145  ],
146  "growth_sweep": [
147    {
148      "d": 2,
149      "epsilon_max": 1.206948960812582,
150      "sweep": [
151        {
152          "fraction_of_max": 0.25,
153          "predicted_layers_to_32": 63,
154          "observed_layers_to_32": 9,
155          "reached": 33
156        },
157        {
158          "fraction_of_max": 0.5,
159          "predicted_layers_to_32": 32,
160          "observed_layers_to_32": 9,
161          "reached": 33
162        },
163        {
164          "fraction_of_max": 0.75,
165          "predicted_layers_to_32": 21,
166          "observed_layers_to_32": 9,
167          "reached": 33
168        },
169        {
170          "fraction_of_max": 1.0,
171          "predicted_layers_to_32": 16,
172          "observed_layers_to_32": 9,
173          "reached": 33
174        }
175      ]
176    },
177    {
178      "d": 4,
179      "epsilon_max": 2.413897921625164,
180      "sweep": [
181        {
182          "fraction_of_max": 0.25,
183          "predicted_layers_to_32": 32,
184          "observed_layers_to_32": 5,
185          "reached": 32
186        },
187        {
188          "fraction_of_max": 0.5,
189          "predicted_layers_to_32": 16,
190          "observed_layers_to_32": 5,
191          "reached": 32
192        },
193        {
194          "fraction_of_max": 0.75,
195          "predicted_layers_to_32": 11,
196          "observed_layers_to_32": 5,
197          "reached": 32
198        },
199        {
200          "fraction_of_max": 1.0,
201          "predicted_layers_to_32": 8,
202          "observed_layers_to_32": 5,
203          "reached": 32
204        }
205      ]
206    },
207    {
208      "d": 6,
209      "epsilon_max": 3.620846882437746,
210      "sweep": [
211        {
212          "fraction_of_max": 0.25,
213          "predicted_layers_to_32": 21,
214          "observed_layers_to_32": 4,
215          "reached": 35
216        },
217        {
218          "fraction_of_max": 0.5,
219          "predicted_layers_to_32": 11,
220          "observed_layers_to_32": 4,
221          "reached": 35
222        },
223        {
224          "fraction_of_max": 0.75,
225          "predicted_layers_to_32": 7,
226          "observed_layers_to_32": 4,
227          "reached": 35
228        },
229        {
230          "fraction_of_max": 1.0,
231          "predicted_layers_to_32": 6,
232          "observed_layers_to_32": 4,
233          "reached": 35
234        }
235      ]
236    },
237    {
238      "d": 8,
239      "epsilon_max": 4.827795843250328,
240      "sweep": [
241        {
242          "fraction_of_max": 0.25,
243          "predicted_layers_to_32": 16,
244          "observed_layers_to_32": 2,
245          "reached": 34
246        },
247        {
248          "fraction_of_max": 0.5,
249          "predicted_layers_to_32": 8,
250          "observed_layers_to_32": 2,
251          "reached": 34
252        },
253        {
254          "fraction_of_max": 0.75,
255          "predicted_layers_to_32": 6,
256          "observed_layers_to_32": 2,
257          "reached": 34
258        },
259        {
260          "fraction_of_max": 1.0,
261          "predicted_layers_to_32": 4,
262          "observed_layers_to_32": 2,
263          "reached": 34
264        }
265      ]
266    }
267  ],
268  "attention_ms": {
269    "dense": 0.3613884999746612,
270    "local": 1.3365088998398278,
271    "expander": 2.6804862999900556
272  },
273  "device": "cuda"
274}