PAC transition-cover training monitor / results.json
Failed on benchmark
1{
2 "seed": 2077,
3 "math": {
4 "scaling_rows": [
5 {
6 "epsilon": 0.02,
7 "n": 10,
8 "observed_miss": 0.816825,
9 "exact": 0.8170728068875467,
10 "exp_bound": 0.8187307530779818,
11 "abs_error_exact": 0.00024780688754666436
12 },
13 {
14 "epsilon": 0.02,
15 "n": 25,
16 "observed_miss": 0.6079,
17 "exact": 0.6034647297788966,
18 "exp_bound": 0.6065306597126334,
19 "abs_error_exact": 0.004435270221103349
20 },
21 {
22 "epsilon": 0.02,
23 "n": 50,
24 "observed_miss": 0.3611,
25 "exact": 0.36416968008711675,
26 "exp_bound": 0.36787944117144233,
27 "abs_error_exact": 0.0030696800871167773
28 },
29 {
30 "epsilon": 0.02,
31 "n": 100,
32 "observed_miss": 0.13435,
33 "exact": 0.13261955589475294,
34 "exp_bound": 0.1353352832366127,
35 "abs_error_exact": 0.0017304441052470587
36 },
37 {
38 "epsilon": 0.02,
39 "n": 200,
40 "observed_miss": 0.017525,
41 "exact": 0.0175879466057215,
42 "exp_bound": 0.01831563888873418,
43 "abs_error_exact": 6.294660572150196e-05
44 },
45 {
46 "epsilon": 0.05,
47 "n": 10,
48 "observed_miss": 0.596875,
49 "exact": 0.5987369392383787,
50 "exp_bound": 0.6065306597126334,
51 "abs_error_exact": 0.0018619392383786293
52 },
53 {
54 "epsilon": 0.05,
55 "n": 25,
56 "observed_miss": 0.278425,
57 "exact": 0.27738957312183377,
58 "exp_bound": 0.2865047968601901,
59 "abs_error_exact": 0.0010354268781662124
60 },
61 {
62 "epsilon": 0.05,
63 "n": 50,
64 "observed_miss": 0.078575,
65 "exact": 0.07694497527671315,
66 "exp_bound": 0.0820849986238988,
67 "abs_error_exact": 0.0016300247232868548
68 },
69 {
70 "epsilon": 0.05,
71 "n": 100,
72 "observed_miss": 0.0062,
73 "exact": 0.0059205292203339975,
74 "exp_bound": 0.006737946999085467,
75 "abs_error_exact": 0.0002794707796660023
76 },
77 {
78 "epsilon": 0.05,
79 "n": 200,
80 "observed_miss": 5e-05,
81 "exact": 3.50526662488287e-05,
82 "exp_bound": 4.5399929762484854e-05,
83 "abs_error_exact": 1.4947333751171303e-05
84 },
85 {
86 "epsilon": 0.1,
87 "n": 10,
88 "observed_miss": 0.3519,
89 "exact": 0.3486784401000001,
90 "exp_bound": 0.36787944117144233,
91 "abs_error_exact": 0.003221559899999893
92 },
93 {
94 "epsilon": 0.1,
95 "n": 25,
96 "observed_miss": 0.07445,
97 "exact": 0.0717897987691853,
98 "exp_bound": 0.0820849986238988,
99 "abs_error_exact": 0.0026602012308147016
100 },
101 {
102 "epsilon": 0.1,
103 "n": 50,
104 "observed_miss": 0.005,
105 "exact": 0.00515377520732012,
106 "exp_bound": 0.006737946999085467,
107 "abs_error_exact": 0.0001537752073201195
108 },
109 {
110 "epsilon": 0.1,
111 "n": 100,
112 "observed_miss": 7.5e-05,
113 "exact": 2.6561398887587544e-05,
114 "exp_bound": 4.5399929762484854e-05,
115 "abs_error_exact": 4.843860111241245e-05
116 },
117 {
118 "epsilon": 0.1,
119 "n": 200,
120 "observed_miss": 0.0,
121 "exact": 7.055079108655367e-10,
122 "exp_bound": 2.061153622438558e-09,
123 "abs_error_exact": 7.055079108655367e-10
124 },
125 {
126 "epsilon": 0.2,
127 "n": 10,
128 "observed_miss": 0.104575,
129 "exact": 0.10737418240000006,
130 "exp_bound": 0.1353352832366127,
131 "abs_error_exact": 0.002799182400000058
132 },
133 {
134 "epsilon": 0.2,
135 "n": 25,
136 "observed_miss": 0.0036,
137 "exact": 0.0037778931862957215,
138 "exp_bound": 0.006737946999085467,
139 "abs_error_exact": 0.00017789318629572162
140 },
141 {
142 "epsilon": 0.2,
143 "n": 50,
144 "observed_miss": 5e-05,
145 "exact": 1.4272476927059638e-05,
146 "exp_bound": 4.5399929762484854e-05,
147 "abs_error_exact": 3.572752307294037e-05
148 },
149 {
150 "epsilon": 0.2,
151 "n": 100,
152 "observed_miss": 0.0,
153 "exact": 2.0370359763344975e-10,
154 "exp_bound": 2.061153622438558e-09,
155 "abs_error_exact": 2.0370359763344975e-10
156 },
157 {
158 "epsilon": 0.2,
159 "n": 200,
160 "observed_miss": 0.0,
161 "exact": 4.149515568881039e-20,
162 "exp_bound": 4.248354255291589e-18,
163 "abs_error_exact": 4.149515568881039e-20
164 }
165 ],
166 "threshold_rows": [
167 {
168 "epsilon": 0.02,
169 "delta": 0.1,
170 "n_required": 116,
171 "observed_miss": 0.096,
172 "target_delta": 0.1,
173 "exact_miss": 0.09598973226261125
174 },
175 {
176 "epsilon": 0.05,
177 "delta": 0.1,
178 "n_required": 47,
179 "observed_miss": 0.090125,
180 "target_delta": 0.1,
181 "exact_miss": 0.08974483193085075
182 },
183 {
184 "epsilon": 0.1,
185 "delta": 0.1,
186 "n_required": 24,
187 "observed_miss": 0.078925,
188 "target_delta": 0.1,
189 "exact_miss": 0.07976644307687256
190 },
191 {
192 "epsilon": 0.2,
193 "delta": 0.1,
194 "n_required": 12,
195 "observed_miss": 0.06835,
196 "target_delta": 0.1,
197 "exact_miss": 0.06871947673600004
198 }
199 ],
200 "max_abs_error_vs_exact": 0.004435270221103349,
201 "threshold_pass": true
202 },
203 "mini_experiment": {
204 "budget": 640,
205 "baseline": {
206 "test_mse": 0.011943768086672121,
207 "cell3_mse": 0.08897263696076005,
208 "rare_branch_mse": 0.6314882576526423,
209 "device": "cuda"
210 },
211 "idea": {
212 "test_mse": 0.0117402829438622,
213 "cell3_mse": 0.0796501513700553,
214 "rare_branch_mse": 0.5347306670579484,
215 "device": "cuda"
216 },
217 "training_cell_counts": {
218 "uniform": [
219 71,
220 86,
221 92,
222 90,
223 74,
224 69,
225 73,
226 85
227 ],
228 "pac": [
229 82,
230 77,
231 79,
232 91,
233 80,
234 76,
235 73,
236 82
237 ]
238 },
239 "pac_monitor": {
240 "required_n": 24,
241 "counts": [
242 82,
243 77,
244 79,
245 91,
246 80,
247 76,
248 73,
249 82
250 ],
251 "deficient": []
252 },
253 "rare_samples": {
254 "uniform": 9,
255 "pac": 10
256 }
257 }
258}