Coordinate-Free BT Monitor for Neural ODEs / bench_report.json
Failed on benchmark
1{
2 "bench_version": 1,
3 "track": "dynamics",
4 "model": "rnn_small",
5 "metric_direction": "lower is better",
6 "n_seeds": 8,
7 "baseline": {
8 "best_cfg": {
9 "lr": 0.006,
10 "weight_decay": 0.0
11 },
12 "sweep": [
13 {
14 "cfg": {
15 "lr": 0.0015,
16 "weight_decay": 0.0
17 },
18 "mean": 0.0020976006926503032
19 },
20 {
21 "cfg": {
22 "lr": 0.003,
23 "weight_decay": 0.0
24 },
25 "mean": 0.0012262430100236088
26 },
27 {
28 "cfg": {
29 "lr": 0.006,
30 "weight_decay": 0.0
31 },
32 "mean": 0.0010091856674989685
33 }
34 ],
35 "full": {
36 "mean": 0.0009940863528754562,
37 "std": 0.0002634151189555855,
38 "per_seed": [
39 0.0007214772049337626,
40 0.0010892449645325541,
41 0.0007245641318149865,
42 0.001501456368714571,
43 0.0007053734152577817,
44 0.0009875509422272444,
45 0.001235381467267871,
46 0.0009876423282548785
47 ],
48 "n": 8
49 }
50 },
51 "idea": {
52 "mean": 0.0009940863528754562,
53 "std": 0.0002634151189555855,
54 "per_seed": [
55 0.0007214772049337626,
56 0.0010892449645325541,
57 0.0007245641318149865,
58 0.001501456368714571,
59 0.0007053734152577817,
60 0.0009875509422272444,
61 0.001235381467267871,
62 0.0009876423282548785
63 ],
64 "n": 8
65 },
66 "comparison": {
67 "delta_mean": 0.0,
68 "idea_wins": 0,
69 "n_pairs": 8,
70 "per_seed_diffs": [
71 0.0,
72 0.0,
73 0.0,
74 0.0,
75 0.0,
76 0.0,
77 0.0,
78 0.0
79 ],
80 "p_value": 1.0,
81 "mde": 0.0,
82 "mde_rel_pct": 0.0,
83 "verdict": "no measurable effect",
84 "system_worked": false
85 },
86 "mechanism_signature": {
87 "quantity": "trained recurrent-map BT coefficients a,b from centered FD along SVD kernel direction",
88 "predicted": "BT penalty reduces fraction of checkpoints/models with |a*b| below threshold",
89 "threshold": 0.0004,
90 "baseline_near_degenerate_fraction": 0.75,
91 "idea_near_degenerate_fraction": 0.75,
92 "per_seed": [
93 {
94 "seed": 0,
95 "baseline": {
96 "a": -0.0035965442657470703,
97 "b": 0.011935234069824219,
98 "ab": 4.2925596062559634e-05,
99 "rank_smin": 0.5489700436592102
100 },
101 "idea": {
102 "a": -0.0035965442657470703,
103 "b": 0.011935234069824219,
104 "ab": 4.2925596062559634e-05,
105 "rank_smin": 0.5489700436592102
106 }
107 },
108 {
109 "seed": 1,
110 "baseline": {
111 "a": 0.01983165740966797,
112 "b": -0.07238030433654785,
113 "ab": 0.0014354214072227478,
114 "rank_smin": 0.4916006326675415
115 },
116 "idea": {
117 "a": 0.01983165740966797,
118 "b": -0.07238030433654785,
119 "ab": 0.0014354214072227478,
120 "rank_smin": 0.4916006326675415
121 }
122 },
123 {
124 "seed": 2,
125 "baseline": {
126 "a": -0.021853148937225342,
127 "b": 0.08207321166992188,
128 "ab": 0.001793558127246797,
129 "rank_smin": 0.48579904437065125
130 },
131 "idea": {
132 "a": -0.021853148937225342,
133 "b": 0.08207321166992188,
134 "ab": 0.001793558127246797,
135 "rank_smin": 0.48579904437065125
136 }
137 },
138 {
139 "seed": 3,
140 "baseline": {
141 "a": -0.00858500599861145,
142 "b": 0.031914710998535156,
143 "ab": 0.0002739879710134119,
144 "rank_smin": 0.4880279004573822
145 },
146 "idea": {
147 "a": -0.00858500599861145,
148 "b": 0.031914710998535156,
149 "ab": 0.0002739879710134119,
150 "rank_smin": 0.4880279004573822
151 }
152 },
153 {
154 "seed": 4,
155 "baseline": {
156 "a": 0.0015027821063995361,
157 "b": -0.0048029422760009766,
158 "ab": 7.217775873868959e-06,
159 "rank_smin": 0.5201539397239685
160 },
161 "idea": {
162 "a": 0.0015027821063995361,
163 "b": -0.0048029422760009766,
164 "ab": 7.217775873868959e-06,
165 "rank_smin": 0.5201539397239685
166 }
167 },
168 {
169 "seed": 5,
170 "baseline": {
171 "a": 0.00692903995513916,
172 "b": -0.02565741539001465,
173 "ab": 0.00017778125766199082,
174 "rank_smin": 0.5120712518692017
175 },
176 "idea": {
177 "a": 0.00692903995513916,
178 "b": -0.02565741539001465,
179 "ab": 0.00017778125766199082,
180 "rank_smin": 0.5120712518692017
181 }
182 },
183 {
184 "seed": 6,
185 "baseline": {
186 "a": 0.0017018616199493408,
187 "b": -0.004662871360778809,
188 "ab": 7.935561370686628e-06,
189 "rank_smin": 0.4283751845359802
190 },
191 "idea": {
192 "a": 0.0017018616199493408,
193 "b": -0.004662871360778809,
194 "ab": 7.935561370686628e-06,
195 "rank_smin": 0.4283751845359802
196 }
197 },
198 {
199 "seed": 7,
200 "baseline": {
201 "a": 0.002718716859817505,
202 "b": -0.008226633071899414,
203 "ab": 2.2365886252373457e-05,
204 "rank_smin": 0.5046995282173157
205 },
206 "idea": {
207 "a": 0.002718716859817505,
208 "b": -0.008226633071899414,
209 "ab": 2.2365886252373457e-05,
210 "rank_smin": 0.5046995282173157
211 }
212 }
213 ],
214 "confirmed": false
215 },
216 "idea_sweep": [
217 {
218 "cfg": {
219 "lr": 0.006,
220 "weight_decay": 0.0
221 },
222 "mean": 0.0009940863528754562,
223 "per_seed": [
224 0.0007214772049337626,
225 0.0010892449645325541,
226 0.0007245641318149865,
227 0.001501456368714571,
228 0.0007053734152577817,
229 0.0009875509422272444,
230 0.001235381467267871,
231 0.0009876423282548785
232 ]
233 },
234 {
235 "cfg": {
236 "lr": 0.0015,
237 "weight_decay": 0.0
238 },
239 "mean": 0.002549259254010394,
240 "per_seed": [
241 0.0012304666452109814,
242 0.003008471569046378,
243 0.001936021144501865,
244 0.0022154434118419886,
245 0.0010678971884772182,
246 0.0025886795483529568,
247 0.004030377604067326,
248 0.00431671692058444
249 ]
250 },
251 {
252 "cfg": {
253 "lr": 0.006,
254 "weight_decay": 0.0
255 },
256 "mean": 0.0009940863528754562,
257 "per_seed": [
258 0.0007214772049337626,
259 0.0010892449645325541,
260 0.0007245641318149865,
261 0.001501456368714571,
262 0.0007053734152577817,
263 0.0009875509422272444,
264 0.001235381467267871,
265 0.0009876423282548785
266 ]
267 }
268 ]
269}