Pick-to-Learn Safety Fine-Tuning / results.json
Failed on benchmark
1{
2 "config": {
3 "C": 0.62,
4 "batch": 32,
5 "rounds": 18,
6 "updates": 12,
7 "beta": 10.0,
8 "lr": 0.045,
9 "temperature": 0.035,
10 "seed": 1217
11 },
12 "prediction_1_order_statistics": [
13 {
14 "B": 4,
15 "observed_max": 0.8015621631577409,
16 "predicted_max": 0.8,
17 "observed_batch_mean": 0.4998967333015562,
18 "predicted_batch_mean": 0.5,
19 "max_minus_mean": 0.30166542985618466
20 },
21 {
22 "B": 8,
23 "observed_max": 0.8892171236640818,
24 "predicted_max": 0.8888888888888888,
25 "observed_batch_mean": 0.5001835241187605,
26 "predicted_batch_mean": 0.5,
27 "max_minus_mean": 0.3890335995453212
28 },
29 {
30 "B": 16,
31 "observed_max": 0.9418454986013614,
32 "predicted_max": 0.9411764705882353,
33 "observed_batch_mean": 0.4994448532502314,
34 "predicted_batch_mean": 0.5,
35 "max_minus_mean": 0.44240064535113
36 },
37 {
38 "B": 32,
39 "observed_max": 0.970142140404991,
40 "predicted_max": 0.9696969696969697,
41 "observed_batch_mean": 0.49959937863476944,
42 "predicted_batch_mean": 0.5,
43 "max_minus_mean": 0.47054276177022164
44 },
45 {
46 "B": 64,
47 "observed_max": 0.9846649293817734,
48 "predicted_max": 0.9846153846153847,
49 "observed_batch_mean": 0.49967429795065593,
50 "predicted_batch_mean": 0.5,
51 "max_minus_mean": 0.48499063143111754
52 },
53 {
54 "B": 128,
55 "observed_max": 0.9922942617176457,
56 "predicted_max": 0.9922480620155039,
57 "observed_batch_mean": 0.4999404400198944,
58 "predicted_batch_mean": 0.5,
59 "max_minus_mean": 0.4923538216977513
60 }
61 ],
62 "prediction_2_equal_budget": {
63 "pick": {
64 "theta": 0.6766023635864258,
65 "violation_rate": 0.08365958170209149,
66 "max_violation": 0.056602363586425786,
67 "mean_violation": 0.0023677149276045946
68 },
69 "uniform": {
70 "theta": 0.8084899187088013,
71 "violation_rate": 0.23313883430582846,
72 "max_violation": 0.18848991870880127,
73 "mean_violation": 0.021972465551050768
74 },
75 "pick_history": [
76 {
77 "round": 1,
78 "theta": 0.6576101779937744,
79 "batch_max_v": 0.3774150430212281,
80 "probe_max_v": 0.03761017322540283
81 },
82 {
83 "round": 2,
84 "theta": 0.6672108769416809,
85 "batch_max_v": 0.01434518410599972,
86 "probe_max_v": 0.047210872173309326
87 },
88 {
89 "round": 3,
90 "theta": 0.6653897166252136,
91 "batch_max_v": 0.040864257912462176,
92 "probe_max_v": 0.04538971185684204
93 },
94 {
95 "round": 4,
96 "theta": 0.6772269606590271,
97 "batch_max_v": 0.0,
98 "probe_max_v": 0.05722695589065552
99 },
100 {
101 "round": 5,
102 "theta": 0.6720226407051086,
103 "batch_max_v": 0.05686082132660497,
104 "probe_max_v": 0.05202263593673706
105 },
106 {
107 "round": 6,
108 "theta": 0.6693175435066223,
109 "batch_max_v": 0.04953771348383296,
110 "probe_max_v": 0.04931753873825073
111 },
112 {
113 "round": 7,
114 "theta": 0.6692908406257629,
115 "batch_max_v": 0.035404469614597756,
116 "probe_max_v": 0.04929083585739136
117 },
118 {
119 "round": 8,
120 "theta": 0.6676847338676453,
121 "batch_max_v": 0.047059099208804245,
122 "probe_max_v": 0.04768472909927368
123 },
124 {
125 "round": 9,
126 "theta": 0.6702789068222046,
127 "batch_max_v": 0.0075303740120028495,
128 "probe_max_v": 0.05027890205383301
129 },
130 {
131 "round": 10,
132 "theta": 0.670312225818634,
133 "batch_max_v": 0.03483712856151411,
134 "probe_max_v": 0.05031222105026245
135 },
136 {
137 "round": 11,
138 "theta": 0.6742766499519348,
139 "batch_max_v": 0.0,
140 "probe_max_v": 0.05427664518356323
141 },
142 {
143 "round": 12,
144 "theta": 0.6754322052001953,
145 "batch_max_v": 0.020520708773855256,
146 "probe_max_v": 0.05543220043182373
147 },
148 {
149 "round": 13,
150 "theta": 0.6735989451408386,
151 "batch_max_v": 0.055137096797685636,
152 "probe_max_v": 0.05359894037246704
153 },
154 {
155 "round": 14,
156 "theta": 0.6733954548835754,
157 "batch_max_v": 0.03763380718439757,
158 "probe_max_v": 0.05339545011520386
159 },
160 {
161 "round": 15,
162 "theta": 0.6745860576629639,
163 "batch_max_v": 0.015337469009649829,
164 "probe_max_v": 0.054586052894592285
165 },
166 {
167 "round": 16,
168 "theta": 0.6758008003234863,
169 "batch_max_v": 0.013176958849893783,
170 "probe_max_v": 0.055800795555114746
171 },
172 {
173 "round": 17,
174 "theta": 0.6762933731079102,
175 "batch_max_v": 0.026348019597901895,
176 "probe_max_v": 0.056293368339538574
177 },
178 {
179 "round": 18,
180 "theta": 0.6766023635864258,
181 "batch_max_v": 0.029256456242445772,
182 "probe_max_v": 0.0566023588180542
183 }
184 ],
185 "uniform_history": [
186 {
187 "round": 1,
188 "theta": 0.8106143474578857,
189 "batch_max_v": 0.3774150430212281,
190 "probe_max_v": 0.19061434268951416
191 },
192 {
193 "round": 2,
194 "theta": 0.7962778210639954,
195 "batch_max_v": 0.16193635786763227,
196 "probe_max_v": 0.17627781629562378
197 },
198 {
199 "round": 3,
200 "theta": 0.8507164716720581,
201 "batch_max_v": 0.16870349614460223,
202 "probe_max_v": 0.23071646690368652
203 },
204 {
205 "round": 4,
206 "theta": 0.8070117235183716,
207 "batch_max_v": 0.16159556234768724,
208 "probe_max_v": 0.18701171875
209 },
210 {
211 "round": 5,
212 "theta": 0.7851780652999878,
213 "batch_max_v": 0.18657541671006272,
214 "probe_max_v": 0.1651780605316162
215 },
216 {
217 "round": 6,
218 "theta": 0.7887235879898071,
219 "batch_max_v": 0.16227472510007224,
220 "probe_max_v": 0.16872358322143555
221 },
222 {
223 "round": 7,
224 "theta": 0.7998988032341003,
225 "batch_max_v": 0.15232842598255825,
226 "probe_max_v": 0.17989879846572876
227 },
228 {
229 "round": 8,
230 "theta": 0.781713604927063,
231 "batch_max_v": 0.17723155130086876,
232 "probe_max_v": 0.1617136001586914
233 },
234 {
235 "round": 9,
236 "theta": 0.8270978927612305,
237 "batch_max_v": 0.1147015829290956,
238 "probe_max_v": 0.2070978879928589
239 },
240 {
241 "round": 10,
242 "theta": 0.8088812828063965,
243 "batch_max_v": 0.1880433437818292,
244 "probe_max_v": 0.1888812780380249
245 },
246 {
247 "round": 11,
248 "theta": 0.8368019461631775,
249 "batch_max_v": 0.1277104284554924,
250 "probe_max_v": 0.2168019413948059
251 },
252 {
253 "round": 12,
254 "theta": 0.8079009652137756,
255 "batch_max_v": 0.1749095904447936,
256 "probe_max_v": 0.18790096044540405
257 },
258 {
259 "round": 13,
260 "theta": 0.7829490900039673,
261 "batch_max_v": 0.1875479788423905,
262 "probe_max_v": 0.1629490852355957
263 },
264 {
265 "round": 14,
266 "theta": 0.8097066879272461,
267 "batch_max_v": 0.14439221677108272,
268 "probe_max_v": 0.1897066831588745
269 },
270 {
271 "round": 15,
272 "theta": 0.8268895745277405,
273 "batch_max_v": 0.14394486184470145,
274 "probe_max_v": 0.2068895697593689
275 },
276 {
277 "round": 16,
278 "theta": 0.7982755303382874,
279 "batch_max_v": 0.15613140704094075,
280 "probe_max_v": 0.17827552556991577
281 },
282 {
283 "round": 17,
284 "theta": 0.870079517364502,
285 "batch_max_v": 0.14348505044776494,
286 "probe_max_v": 0.25007951259613037
287 },
288 {
289 "round": 18,
290 "theta": 0.8084899187088013,
291 "batch_max_v": 0.2152954007181691,
292 "probe_max_v": 0.1884899139404297
293 }
294 ],
295 "rollouts_each": 576
296 },
297 "prediction_3_certificate": {
298 "rows": [
299 {
300 "N": 50,
301 "phat": 0.16,
302 "upper": 0.3330818382602285,
303 "epsilon": 0.17308183826022852
304 },
305 {
306 "N": 100,
307 "phat": 0.02,
308 "upper": 0.1423873415340408,
309 "epsilon": 0.12238734153404082
310 },
311 {
312 "N": 200,
313 "phat": 0.07,
314 "upper": 0.15654091913011425,
315 "epsilon": 0.08654091913011426
316 },
317 {
318 "N": 400,
319 "phat": 0.07,
320 "upper": 0.13119367076702043,
321 "epsilon": 0.06119367076702041
322 },
323 {
324 "N": 600,
325 "phat": 0.08666666666666667,
326 "upper": 0.13663108962235576,
327 "epsilon": 0.049964422955689106
328 },
329 {
330 "N": 1200,
331 "phat": 0.08083333333333333,
332 "upper": 0.1161635156233739,
333 "epsilon": 0.03533018229004057
334 },
335 {
336 "N": 2400,
337 "phat": 0.08083333333333333,
338 "upper": 0.10581554481117789,
339 "epsilon": 0.024982211477844553
340 }
341 ],
342 "observed_loglog_slope": -0.49999999999999994,
343 "predicted_slope": -0.5,
344 "heldout_600": {
345 "N": 600,
346 "violations": 52,
347 "phat": 0.08666666666666667,
348 "upper_95": 0.13663108962235576
349 }
350 },
351 "notes": "Violation is g=[theta*z-C], with U[0,1] severity and fixed policy during certification."
352}