Distributed E-Value Prediction Sets / results.json
Mechanism confirmed, baseline not beaten
1{
2 "seed": 939,
3 "n_cal": 12000,
4 "n_test": 50000,
5 "alpha": 0.1,
6 "construction": "inverse_probability_e_value",
7 "uncertainty": [
8 1.0,
9 0.72,
10 0.48,
11 0.28,
12 0.12
13 ],
14 "normalizers": [
15 1.2982767982459749,
16 0.7764660205469652,
17 0.510018497995707,
18 0.39457924969358227,
19 0.3253161916328093
20 ],
21 "local_calibration_true_e_means": [
22 0.9900990099009908,
23 0.9900990099009968,
24 0.9900990099009944,
25 0.9900990099009905,
26 0.9900990099009895
27 ],
28 "local_test_true_e_means": [
29 0.9850652832875366,
30 0.9962832981771158,
31 1.000623370373759,
32 0.9910507325125879,
33 0.9899601103082599
34 ],
35 "sweeps": {
36 "kappa": [
37 {
38 "mean_fused_true_e": 0.992596558931852,
39 "coverage": 0.99954,
40 "rejection": 0.00046,
41 "markov_bound": 0.0992596558931852,
42 "avg_set_size": 3.56554,
43 "singleton_rate": 0.05208,
44 "point_accuracy": 0.9926,
45 "kappa": 0.0,
46 "weights": [
47 0.2,
48 0.2,
49 0.2,
50 0.2,
51 0.2
52 ],
53 "predicted_mean": 0.9925965589318518,
54 "convexity_abs_error": 1.1102230246251565e-16
55 },
56 {
57 "mean_fused_true_e": 0.9927225274672686,
58 "coverage": 0.99974,
59 "rejection": 0.00026,
60 "markov_bound": 0.09927225274672687,
61 "avg_set_size": 3.40926,
62 "singleton_rate": 0.06974,
63 "point_accuracy": 0.99522,
64 "kappa": 0.5,
65 "weights": [
66 0.15544265710408217,
67 0.17880161569157396,
68 0.20159825874955517,
69 0.22280053270469782,
70 0.24135693575009082
71 ],
72 "predicted_mean": 0.9927225274672687,
73 "convexity_abs_error": 1.1102230246251565e-16
74 },
75 {
76 "mean_fused_true_e": 0.9927519297947519,
77 "coverage": 0.9999,
78 "rejection": 0.0001,
79 "markov_bound": 0.0992751929794752,
80 "avg_set_size": 3.27532,
81 "singleton_rate": 0.08672,
82 "point_accuracy": 0.99678,
83 "kappa": 1.0,
84 "weights": [
85 0.11805691940429969,
86 0.15620462961654696,
87 0.19857500267630504,
88 0.24254005597050207,
89 0.2846233923323463
90 ],
91 "predicted_mean": 0.992751929794752,
92 "convexity_abs_error": 1.1102230246251565e-16
93 },
94 {
95 "mean_fused_true_e": 0.9925827200485682,
96 "coverage": 0.99996,
97 "rejection": 4e-05,
98 "markov_bound": 0.09925827200485682,
99 "avg_set_size": 3.06882,
100 "singleton_rate": 0.11758,
101 "point_accuracy": 0.9981,
102 "kappa": 2.0,
103 "weights": [
104 0.0640490965488525,
105 0.11212899199688599,
106 0.18120879370985926,
107 0.2703317538861501,
108 0.3722813638582521
109 ],
110 "predicted_mean": 0.9925827200485682,
111 "convexity_abs_error": 0.0
112 },
113 {
114 "mean_fused_true_e": 0.9914555491371337,
115 "coverage": 1.0,
116 "rejection": 0.0,
117 "markov_bound": 0.09914555491371338,
118 "avg_set_size": 2.83658,
119 "singleton_rate": 0.1614,
120 "point_accuracy": 0.99726,
121 "kappa": 5.0,
122 "weights": [
123 0.0073223573529251585,
124 0.029693623294806973,
125 0.09858630119845277,
126 0.2679853510827443,
127 0.5964123670710708
128 ],
129 "predicted_mean": 0.9914555491371337,
130 "convexity_abs_error": 0.0
131 }
132 ],
133 "active_experts": [
134 {
135 "mean_fused_true_e": 0.9850652832875366,
136 "coverage": 0.99272,
137 "rejection": 0.00728,
138 "markov_bound": 0.09850652832875367,
139 "avg_set_size": 4.56496,
140 "singleton_rate": 0.00366,
141 "point_accuracy": 0.4917,
142 "n_active": 1,
143 "weights": [
144 1.0
145 ],
146 "predicted_mean": 0.9850652832875366
147 },
148 {
149 "mean_fused_true_e": 0.9922050173609256,
150 "coverage": 0.9975,
151 "rejection": 0.0025,
152 "markov_bound": 0.09922050173609256,
153 "avg_set_size": 4.40864,
154 "singleton_rate": 0.00602,
155 "point_accuracy": 0.71094,
156 "n_active": 2,
157 "weights": [
158 0.36354745971843366,
159 0.6364525402815663
160 ],
161 "predicted_mean": 0.9922050173609257
162 },
163 {
164 "mean_fused_true_e": 0.9964734437364307,
165 "coverage": 0.99922,
166 "rejection": 0.00078,
167 "markov_bound": 0.09964734437364307,
168 "avg_set_size": 4.13592,
169 "singleton_rate": 0.01474,
170 "point_accuracy": 0.88606,
171 "n_active": 3,
172 "weights": [
173 0.17921501803484088,
174 0.3137468037136657,
175 0.5070381782514934
176 ],
177 "predicted_mean": 0.9964734437364307
178 },
179 {
180 "mean_fused_true_e": 0.9941381123450133,
181 "coverage": 0.9999,
182 "rejection": 0.0001,
183 "markov_bound": 0.09941381123450133,
184 "avg_set_size": 3.69836,
185 "singleton_rate": 0.04176,
186 "point_accuracy": 0.97654,
187 "n_active": 4,
188 "weights": [
189 0.10203472202534593,
190 0.17862938192513,
191 0.2886783716087405,
192 0.43065752444078353
193 ],
194 "predicted_mean": 0.9941381123450133
195 },
196 {
197 "mean_fused_true_e": 0.9925827200485682,
198 "coverage": 0.99996,
199 "rejection": 4e-05,
200 "markov_bound": 0.09925827200485682,
201 "avg_set_size": 3.06882,
202 "singleton_rate": 0.11758,
203 "point_accuracy": 0.9981,
204 "n_active": 5,
205 "weights": [
206 0.0640490965488525,
207 0.11212899199688599,
208 0.18120879370985926,
209 0.2703317538861501,
210 0.3722813638582521
211 ],
212 "predicted_mean": 0.9925827200485682
213 }
214 ],
215 "alpha": [
216 {
217 "mean_fused_true_e": 0.9925827200485682,
218 "coverage": 1.0,
219 "rejection": 0.0,
220 "markov_bound": 0.019851654400971364,
221 "avg_set_size": 4.96354,
222 "singleton_rate": 2e-05,
223 "point_accuracy": 0.9981,
224 "alpha": 0.02,
225 "weights": [
226 0.0640490965488525,
227 0.11212899199688599,
228 0.18120879370985926,
229 0.2703317538861501,
230 0.3722813638582521
231 ]
232 },
233 {
234 "mean_fused_true_e": 0.9925827200485682,
235 "coverage": 1.0,
236 "rejection": 0.0,
237 "markov_bound": 0.04962913600242841,
238 "avg_set_size": 4.4901,
239 "singleton_rate": 0.006,
240 "point_accuracy": 0.9981,
241 "alpha": 0.05,
242 "weights": [
243 0.0640490965488525,
244 0.11212899199688599,
245 0.18120879370985926,
246 0.2703317538861501,
247 0.3722813638582521
248 ]
249 },
250 {
251 "mean_fused_true_e": 0.9925827200485682,
252 "coverage": 0.99996,
253 "rejection": 4e-05,
254 "markov_bound": 0.09925827200485682,
255 "avg_set_size": 3.06882,
256 "singleton_rate": 0.11758,
257 "point_accuracy": 0.9981,
258 "alpha": 0.1,
259 "weights": [
260 0.0640490965488525,
261 0.11212899199688599,
262 0.18120879370985926,
263 0.2703317538861501,
264 0.3722813638582521
265 ]
266 },
267 {
268 "mean_fused_true_e": 0.9925827200485682,
269 "coverage": 0.99954,
270 "rejection": 0.00046,
271 "markov_bound": 0.19851654400971364,
272 "avg_set_size": 1.46222,
273 "singleton_rate": 0.6429,
274 "point_accuracy": 0.9981,
275 "alpha": 0.2,
276 "weights": [
277 0.0640490965488525,
278 0.11212899199688599,
279 0.18120879370985926,
280 0.2703317538861501,
281 0.3722813638582521
282 ]
283 },
284 {
285 "mean_fused_true_e": 0.9925827200485682,
286 "coverage": 0.9941,
287 "rejection": 0.0059,
288 "markov_bound": 0.39703308801942727,
289 "avg_set_size": 1.01084,
290 "singleton_rate": 0.97824,
291 "point_accuracy": 0.9981,
292 "alpha": 0.4,
293 "weights": [
294 0.0640490965488525,
295 0.11212899199688599,
296 0.18120879370985926,
297 0.2703317538861501,
298 0.3722813638582521
299 ]
300 }
301 ]
302 },
303 "baseline": {
304 "strongest_expert_point_accuracy": 0.9659,
305 "strongest_expert_coverage": 0.99994,
306 "strongest_expert_avg_set_size": 2.92814,
307 "unweighted": {
308 "mean_fused_true_e": 0.992596558931852,
309 "coverage": 0.99954,
310 "rejection": 0.00046,
311 "markov_bound": 0.0992596558931852,
312 "avg_set_size": 3.56554,
313 "singleton_rate": 0.05208,
314 "point_accuracy": 0.9926
315 },
316 "uncertainty_weighted": {
317 "mean_fused_true_e": 0.9925827200485682,
318 "coverage": 0.99996,
319 "rejection": 4e-05,
320 "markov_bound": 0.09925827200485682,
321 "avg_set_size": 3.06882,
322 "singleton_rate": 0.11758,
323 "point_accuracy": 0.9981
324 }
325 }
326}