Residual-Scenario Safety Training / artifacts/bench_report.json
Failed on benchmark
1{
2 "bench_version": 1,
3 "track": "dynamics",
4 "model": "rnn_small",
5 "metric_direction": "lower is better",
6 "n_seeds": 8,
7 "baseline": {
8 "best_cfg": {
9 "lr": 0.01,
10 "mu": 0.0
11 },
12 "sweep": [
13 {
14 "cfg": {
15 "lr": 0.001,
16 "mu": 0.0
17 },
18 "mean": 0.004298779065720737
19 },
20 {
21 "cfg": {
22 "lr": 0.001,
23 "mu": 0.05
24 },
25 "mean": 0.004298779065720737
26 },
27 {
28 "cfg": {
29 "lr": 0.001,
30 "mu": 0.2
31 },
32 "mean": 0.004298779065720737
33 },
34 {
35 "cfg": {
36 "lr": 0.001,
37 "mu": 0.5
38 },
39 "mean": 0.004298779065720737
40 },
41 {
42 "cfg": {
43 "lr": 0.003,
44 "mu": 0.0
45 },
46 "mean": 0.0034615940821822733
47 },
48 {
49 "cfg": {
50 "lr": 0.003,
51 "mu": 0.05
52 },
53 "mean": 0.0034615940821822733
54 },
55 {
56 "cfg": {
57 "lr": 0.003,
58 "mu": 0.2
59 },
60 "mean": 0.0034615940821822733
61 },
62 {
63 "cfg": {
64 "lr": 0.003,
65 "mu": 0.5
66 },
67 "mean": 0.0034615940821822733
68 },
69 {
70 "cfg": {
71 "lr": 0.01,
72 "mu": 0.0
73 },
74 "mean": 0.0014865802077110857
75 },
76 {
77 "cfg": {
78 "lr": 0.01,
79 "mu": 0.05
80 },
81 "mean": 0.0014865802077110857
82 },
83 {
84 "cfg": {
85 "lr": 0.01,
86 "mu": 0.2
87 },
88 "mean": 0.0014865802077110857
89 },
90 {
91 "cfg": {
92 "lr": 0.01,
93 "mu": 0.5
94 },
95 "mean": 0.0014865802077110857
96 }
97 ],
98 "full": {
99 "mean": 0.0014198302378645167,
100 "std": 0.00039839114901047844,
101 "per_seed": [
102 0.0009327387670055032,
103 0.0018641706556081772,
104 0.0010785290505737066,
105 0.0020708823576569557,
106 0.0009260809747502208,
107 0.0014055331703275442,
108 0.0016443432541564107,
109 0.001436363672837615
110 ],
111 "n": 8
112 }
113 },
114 "idea": {
115 "mean": 0.0013042011414654553,
116 "std": 0.0005283693045785661,
117 "per_seed": [
118 0.0023362392093986273,
119 0.0014533286448568106,
120 0.0007090693688951433,
121 0.0012600651243701577,
122 0.0010053017176687717,
123 0.001083051203750074,
124 0.0007225603912957013,
125 0.0018639934714883566
126 ],
127 "n": 8,
128 "best_cfg": {
129 "lr": 0.01,
130 "mu": 0.05
131 },
132 "sweep": [
133 {
134 "cfg": {
135 "lr": 0.001,
136 "mu": 0.05
137 },
138 "mean": 0.004056878213305026
139 },
140 {
141 "cfg": {
142 "lr": 0.001,
143 "mu": 0.2
144 },
145 "mean": 0.004066162509843707
146 },
147 {
148 "cfg": {
149 "lr": 0.001,
150 "mu": 0.5
151 },
152 "mean": 0.004089910595212132
153 },
154 {
155 "cfg": {
156 "lr": 0.003,
157 "mu": 0.05
158 },
159 "mean": 0.003846683946903795
160 },
161 {
162 "cfg": {
163 "lr": 0.003,
164 "mu": 0.2
165 },
166 "mean": 0.003857339732348919
167 },
168 {
169 "cfg": {
170 "lr": 0.003,
171 "mu": 0.5
172 },
173 "mean": 0.0038832707796245813
174 },
175 {
176 "cfg": {
177 "lr": 0.01,
178 "mu": 0.05
179 },
180 "mean": 0.0014396755868801847
181 },
182 {
183 "cfg": {
184 "lr": 0.01,
185 "mu": 0.2
186 },
187 "mean": 0.0014551622880389914
188 },
189 {
190 "cfg": {
191 "lr": 0.01,
192 "mu": 0.5
193 },
194 "mean": 0.001473813972552307
195 }
196 ]
197 },
198 "comparison": {
199 "delta_mean": -0.00011562909639906138,
200 "idea_wins": 5,
201 "n_pairs": 8,
202 "per_seed_diffs": [
203 0.001403500442393124,
204 -0.0004108420107513666,
205 -0.00036945968167856336,
206 -0.000810817233286798,
207 7.922074291855097e-05,
208 -0.0003224819665774703,
209 -0.0009217828628607094,
210 0.0004276297986507416
211 ],
212 "p_value": 0.6789,
213 "mde": 0.0006295704392970048,
214 "mde_rel_pct": 44.341247460957355,
215 "verdict": "no significant win",
216 "system_worked": false
217 },
218 "track_justification": "Dynamics is structurally matched: the task is controlled pendulum rollout and the intervention constrains predicted outputs under empirical error scenarios.",
219 "idea_sweep": [
220 {
221 "cfg": {
222 "lr": 0.001,
223 "mu": 0.05
224 },
225 "mean": 0.004056878213305026
226 },
227 {
228 "cfg": {
229 "lr": 0.001,
230 "mu": 0.2
231 },
232 "mean": 0.004066162509843707
233 },
234 {
235 "cfg": {
236 "lr": 0.001,
237 "mu": 0.5
238 },
239 "mean": 0.004089910595212132
240 },
241 {
242 "cfg": {
243 "lr": 0.003,
244 "mu": 0.05
245 },
246 "mean": 0.003846683946903795
247 },
248 {
249 "cfg": {
250 "lr": 0.003,
251 "mu": 0.2
252 },
253 "mean": 0.003857339732348919
254 },
255 {
256 "cfg": {
257 "lr": 0.003,
258 "mu": 0.5
259 },
260 "mean": 0.0038832707796245813
261 },
262 {
263 "cfg": {
264 "lr": 0.01,
265 "mu": 0.05
266 },
267 "mean": 0.0014396755868801847
268 },
269 {
270 "cfg": {
271 "lr": 0.01,
272 "mu": 0.2
273 },
274 "mean": 0.0014551622880389914
275 },
276 {
277 "cfg": {
278 "lr": 0.01,
279 "mu": 0.5
280 },
281 "mean": 0.001473813972552307
282 }
283 ],
284 "custom_track": null,
285 "mechanism_signature": {
286 "baseline_residual_std": 0.034058332443237305,
287 "idea_residual_std": 0.031945905182510614,
288 "baseline_mean_slack": 0.0,
289 "idea_mean_slack": 0.0,
290 "prediction": "scenario penalty reduces empirical residual-boundary slack",
291 "confirmed": false
292 }
293}