Pick-to-Learn Safety Fine-Tuning / bench_report.json
Failed on benchmark
1{
2 "bench_version": 1,
3 "track": "dynamics",
4 "model": "rnn_small",
5 "metric_direction": "lower is better",
6 "n_seeds": 8,
7 "baseline": {
8 "best_cfg": {
9 "lr": 0.01,
10 "weight_decay": 0.0
11 },
12 "sweep": [
13 {
14 "cfg": {
15 "lr": 0.001,
16 "weight_decay": 0.0
17 },
18 "mean": 0.0006690263253403828
19 },
20 {
21 "cfg": {
22 "lr": 0.003,
23 "weight_decay": 0.0
24 },
25 "mean": 0.0001483298638049746
26 },
27 {
28 "cfg": {
29 "lr": 0.01,
30 "weight_decay": 0.0
31 },
32 "mean": 2.2257129785430152e-05
33 }
34 ],
35 "full": {
36 "mean": 2.8511603204606217e-05,
37 "std": 8.769606779676738e-06,
38 "per_seed": [
39 1.911987055791542e-05,
40 1.707040792098269e-05,
41 2.927937930508051e-05,
42 2.355886135774199e-05,
43 3.0128363505355082e-05,
44 2.610497540445067e-05,
45 3.756858495762572e-05,
46 4.526238262769766e-05
47 ],
48 "n": 8
49 }
50 },
51 "idea": {
52 "mean": 0.10648404061794281,
53 "std": 0.01916183881366529,
54 "per_seed": [
55 0.08292833715677261,
56 0.14024589955806732,
57 0.11994219571352005,
58 0.12405627220869064,
59 0.10991806536912918,
60 0.08748061209917068,
61 0.08862826228141785,
62 0.09867268055677414
63 ],
64 "n": 8
65 },
66 "comparison": {
67 "delta_mean": 0.1064555290147382,
68 "idea_wins": 0,
69 "n_pairs": 8,
70 "per_seed_diffs": [
71 0.0829092172862147,
72 0.14022882915014634,
73 0.11991291633421497,
74 0.1240327133473329,
75 0.10988793700562383,
76 0.08745450712376623,
77 0.08859069369646022,
78 0.09862741817414644
79 ],
80 "p_value": 0.0081,
81 "mde": 0.017131317197169284,
82 "mde_rel_pct": 60085.42232518731,
83 "verdict": "idea worse (significant)",
84 "system_worked": false
85 },
86 "mechanism_signature": {
87 "safety_limit": 0.8,
88 "scale": 0.25,
89 "rows": [
90 {
91 "seed": 0,
92 "baseline_pred_rate": 0.41,
93 "idea_pred_rate": 0.25333333333333335,
94 "observed_rate": 0.41,
95 "baseline_pred_max_margin": 1.1175339221954346,
96 "idea_pred_max_margin": 0.0780906081199646
97 },
98 {
99 "seed": 1,
100 "baseline_pred_rate": 0.43,
101 "idea_pred_rate": 0.0,
102 "observed_rate": 0.43,
103 "baseline_pred_max_margin": 1.0835604667663574,
104 "idea_pred_max_margin": 0.0
105 },
106 {
107 "seed": 2,
108 "baseline_pred_rate": 0.43666666666666665,
109 "idea_pred_rate": 0.03333333333333333,
110 "observed_rate": 0.43666666666666665,
111 "baseline_pred_max_margin": 1.0742902755737305,
112 "idea_pred_max_margin": 0.01500558853149414
113 },
114 {
115 "seed": 3,
116 "baseline_pred_rate": 0.43666666666666665,
117 "idea_pred_rate": 0.31666666666666665,
118 "observed_rate": 0.43333333333333335,
119 "baseline_pred_max_margin": 1.1583514213562012,
120 "idea_pred_max_margin": 0.08182209730148315
121 },
122 {
123 "seed": 4,
124 "baseline_pred_rate": 0.42333333333333334,
125 "idea_pred_rate": 0.14333333333333334,
126 "observed_rate": 0.42333333333333334,
127 "baseline_pred_max_margin": 1.150449514389038,
128 "idea_pred_max_margin": 0.07036739587783813
129 },
130 {
131 "seed": 5,
132 "baseline_pred_rate": 0.4166666666666667,
133 "idea_pred_rate": 0.18666666666666668,
134 "observed_rate": 0.4166666666666667,
135 "baseline_pred_max_margin": 1.0507187843322754,
136 "idea_pred_max_margin": 0.08430737257003784
137 },
138 {
139 "seed": 6,
140 "baseline_pred_rate": 0.39,
141 "idea_pred_rate": 0.22,
142 "observed_rate": 0.39,
143 "baseline_pred_max_margin": 1.1166901588439941,
144 "idea_pred_max_margin": 0.07752972841262817
145 },
146 {
147 "seed": 7,
148 "baseline_pred_rate": 0.4,
149 "idea_pred_rate": 0.11666666666666667,
150 "observed_rate": 0.39666666666666667,
151 "baseline_pred_max_margin": 1.1013514995574951,
152 "idea_pred_max_margin": 0.013077378273010254
153 }
154 ],
155 "observed_rate_mean": 0.41708333333333336,
156 "predicted_rate_baseline_mean": 0.4179166666666667,
157 "predicted_rate_idea_mean": 0.15875,
158 "predicted_reduction_fraction": 0.6201395812562314,
159 "observed_reduction_fraction": 0.0,
160 "confirmed": false
161 },
162 "idea_sweep": [
163 {
164 "cfg": {
165 "lr": 0.001,
166 "weight_decay": 0.0
167 },
168 "result": {
169 "mean": 0.11110909003764391,
170 "std": 0.013193315288929698,
171 "per_seed": [
172 0.09354278445243835,
173 0.13219793140888214,
174 0.11154640465974808,
175 0.1305529773235321,
176 0.10504380613565445,
177 0.09625166654586792,
178 0.10946251451969147,
179 0.11027463525533676
180 ],
181 "n": 8
182 }
183 },
184 {
185 "cfg": {
186 "lr": 0.003,
187 "weight_decay": 0.0
188 },
189 "result": {
190 "mean": 0.11486770864576101,
191 "std": 0.010136738239810605,
192 "per_seed": [
193 0.11077549308538437,
194 0.1312650591135025,
195 0.11508128046989441,
196 0.12773503363132477,
197 0.11910589039325714,
198 0.0991341769695282,
199 0.11002153158187866,
200 0.10582320392131805
201 ],
202 "n": 8
203 }
204 },
205 {
206 "cfg": {
207 "lr": 0.01,
208 "weight_decay": 0.0
209 },
210 "result": {
211 "mean": 0.10648404061794281,
212 "std": 0.01916183881366529,
213 "per_seed": [
214 0.08292833715677261,
215 0.14024589955806732,
216 0.11994219571352005,
217 0.12405627220869064,
218 0.10991806536912918,
219 0.08748061209917068,
220 0.08862826228141785,
221 0.09867268055677414
222 ],
223 "n": 8
224 }
225 }
226 ],
227 "protocol": {
228 "seeds": [
229 0,
230 1,
231 2,
232 3,
233 4,
234 5,
235 6,
236 7
237 ],
238 "n_train": 1000,
239 "n_test": 300,
240 "epochs": 18,
241 "batch": 64,
242 "structural_match": "dynamics: controlled pendulum rollout and stability/safety violations",
243 "baseline_method": "uniform per-example MSE",
244 "idea_method": "top-8 normalized predicted-angle violation replay"
245 }
246}