Critical-Slowing-Down Safety Monitor / bench_report.json
Failed on benchmark
1{
2 "bench_version": 1,
3 "track": "dynamics",
4 "model": "rnn_small",
5 "metric_direction": "lower is better",
6 "n_seeds": 8,
7 "baseline": {
8 "best_cfg": {
9 "lr": 0.006,
10 "weight_decay": 0.0
11 },
12 "sweep": [
13 {
14 "cfg": {
15 "lr": 0.0015,
16 "weight_decay": 0.0
17 },
18 "mean": 0.0017880297382362187
19 },
20 {
21 "cfg": {
22 "lr": 0.003,
23 "weight_decay": 0.0
24 },
25 "mean": 0.0012683377135545015
26 },
27 {
28 "cfg": {
29 "lr": 0.006,
30 "weight_decay": 0.0
31 },
32 "mean": 0.0008017216168809682
33 }
34 ],
35 "full": {
36 "mean": 0.0007909468331490643,
37 "std": 0.00016569300329491533,
38 "per_seed": [
39 0.0008831982268020511,
40 0.0008114342344924808,
41 0.0005164801841601729,
42 0.000995773822069168,
43 0.0005416677449829876,
44 0.0009393909713253379,
45 0.0007582727121189237,
46 0.0008813567692413926
47 ],
48 "n": 8
49 }
50 },
51 "idea": {
52 "mean": 0.0008200057927751914,
53 "std": 0.0001585617147958542,
54 "per_seed": [
55 0.0008831982268020511,
56 0.0008114342344924808,
57 0.0005565483006648719,
58 0.0009936458664014935,
59 0.0005761774955317378,
60 0.0009393909713253379,
61 0.0008254630374722183,
62 0.0009741882095113397
63 ],
64 "n": 8,
65 "best_cfg": {
66 "lr": 0.006,
67 "weight_decay": 0.0
68 },
69 "sweep": [
70 {
71 "cfg": {
72 "lr": 0.0015,
73 "weight_decay": 0.0
74 },
75 "mean": 0.00219198341073934,
76 "per_seed": [
77 0.0010475046001374722,
78 0.0022253927309066057,
79 0.0021842154674232006,
80 0.00284200394526124,
81 0.001567519037052989,
82 0.001590907690115273,
83 0.0031952967401593924,
84 0.0028830270748585463
85 ]
86 },
87 {
88 "cfg": {
89 "lr": 0.003,
90 "weight_decay": 0.0
91 },
92 "mean": 0.001621349343622569,
93 "per_seed": [
94 0.0009604159276932478,
95 0.0015390310436487198,
96 0.0013559971703216434,
97 0.0021366437431424856,
98 0.0009070083615370095,
99 0.00200911913998425,
100 0.0018186086090281606,
101 0.0022439707536250353
102 ]
103 },
104 {
105 "cfg": {
106 "lr": 0.006,
107 "weight_decay": 0.0
108 },
109 "mean": 0.0008200057927751914,
110 "per_seed": [
111 0.0008831982268020511,
112 0.0008114342344924808,
113 0.0005565483006648719,
114 0.0009936458664014935,
115 0.0005761774955317378,
116 0.0009393909713253379,
117 0.0008254630374722183,
118 0.0009741882095113397
119 ]
120 }
121 ]
122 },
123 "comparison": {
124 "delta_mean": 2.9058959626127034e-05,
125 "idea_wins": 1,
126 "n_pairs": 8,
127 "per_seed_diffs": [
128 0.0,
129 0.0,
130 4.006811650469899e-05,
131 -2.1279556676745415e-06,
132 3.450975054875016e-05,
133 0.0,
134 6.719032535329461e-05,
135 9.283144026994705e-05
136 ],
137 "p_value": 0.12595,
138 "mde": 3.027644738730695e-05,
139 "mde_rel_pct": 3.8278738997872637,
140 "verdict": "no significant win",
141 "system_worked": false
142 },
143 "mechanism_signature": {
144 "math_check": {
145 "rows": [
146 {
147 "a": 0.2,
148 "predicted_variance": 1.0416666666666667,
149 "observed_variance": 1.0342422123173947,
150 "predicted_rho": 0.2,
151 "observed_rho": 0.20071559852735493,
152 "variance_relative_error": 0.007127476175301197,
153 "rho_abs_error": 0.0007155985273549215
154 },
155 {
156 "a": 0.5,
157 "predicted_variance": 1.3333333333333333,
158 "observed_variance": 1.3261153307140452,
159 "predicted_rho": 0.5,
160 "observed_rho": 0.5016303562384845,
161 "variance_relative_error": 0.00541350196446605,
162 "rho_abs_error": 0.0016303562384845094
163 },
164 {
165 "a": 0.7,
166 "predicted_variance": 1.9607843137254901,
167 "observed_variance": 1.9111379220646099,
168 "predicted_rho": 0.7,
169 "observed_rho": 0.6959265998902273,
170 "variance_relative_error": 0.02531965974704893,
171 "rho_abs_error": 0.00407340010977264
172 },
173 {
174 "a": 0.85,
175 "predicted_variance": 3.6036036036036028,
176 "observed_variance": 3.595122775671891,
177 "predicted_rho": 0.85,
178 "observed_rho": 0.8512361072857468,
179 "variance_relative_error": 0.0023534297510499906,
180 "rho_abs_error": 0.001236107285746857
181 },
182 {
183 "a": 0.93,
184 "predicted_variance": 7.401924500370103,
185 "observed_variance": 7.427471227984546,
186 "predicted_rho": 0.93,
187 "observed_rho": 0.9301212269556333,
188 "variance_relative_error": 0.0034513629007112147,
189 "rho_abs_error": 0.0001212269556332135
190 }
191 ],
192 "max_variance_relative_error": 0.02531965974704893,
193 "max_rho_abs_error": 0.00407340010977264
194 },
195 "observable": "trained rnn_small minibatch gradient norm",
196 "prediction": "joint high AR(1) and rising variance precede LR intervention",
197 "observed": [
198 {
199 "seed": 0,
200 "max_a_hat": 0.7482613680202572,
201 "variance_slope": -0.0044005689645504,
202 "interventions": 0
203 },
204 {
205 "seed": 1,
206 "max_a_hat": 0.7772662940130003,
207 "variance_slope": -0.0034576642656289226,
208 "interventions": 0
209 },
210 {
211 "seed": 2,
212 "max_a_hat": 0.7882554383146971,
213 "variance_slope": -0.002863924976655482,
214 "interventions": 1
215 },
216 {
217 "seed": 3,
218 "max_a_hat": 0.7757743698679345,
219 "variance_slope": -0.003068314824752291,
220 "interventions": 1
221 },
222 {
223 "seed": 4,
224 "max_a_hat": 0.8173354197062572,
225 "variance_slope": -0.0021798070528012223,
226 "interventions": 3
227 },
228 {
229 "seed": 5,
230 "max_a_hat": 0.7417832255136816,
231 "variance_slope": -0.003818343265481793,
232 "interventions": 0
233 },
234 {
235 "seed": 6,
236 "max_a_hat": 0.8150579419646545,
237 "variance_slope": -0.0023863047120519687,
238 "interventions": 1
239 },
240 {
241 "seed": 7,
242 "max_a_hat": 1.0173393024141661,
243 "variance_slope": -0.0019478986432107611,
244 "interventions": 12
245 }
246 ],
247 "mean_max_a_hat": 0.810134169976831,
248 "mean_variance_slope": -0.003015353338141605,
249 "confirmed": false
250 },
251 "protocol_notes": {
252 "paired_seeds": [
253 0,
254 1,
255 2,
256 3,
257 4,
258 5,
259 6,
260 7
261 ],
262 "dataset_sizes": [
263 600,
264 240
265 ],
266 "idea_lr_union": [
267 0.0015,
268 0.003,
269 0.006
270 ],
271 "baseline_lr_union": [
272 0.0015,
273 0.003,
274 0.006
275 ],
276 "track_reason": "dynamics structurally matches stability/control monitor"
277 }
278}