Exact Event-Chained Neural ODE / bench_report.json
Unverified
1{
2 "bench_version": 1,
3 "track": "dynamics",
4 "model": "rnn_small",
5 "metric_direction": "lower is better",
6 "n_seeds": 8,
7 "baseline": {
8 "best_cfg": {
9 "lr": 0.006,
10 "weight_decay": 0.0
11 },
12 "sweep": [
13 {
14 "cfg": {
15 "lr": 0.0015,
16 "weight_decay": 0.0
17 },
18 "mean": 3.7836247429368086e-05
19 },
20 {
21 "cfg": {
22 "lr": 0.003,
23 "weight_decay": 0.0
24 },
25 "mean": 1.23220872865204e-05
26 },
27 {
28 "cfg": {
29 "lr": 0.006,
30 "weight_decay": 0.0
31 },
32 "mean": 7.367291686932731e-06
33 }
34 ],
35 "full": {
36 "mean": 8.597878206728637e-06,
37 "std": 2.099193175555627e-06,
38 "per_seed": [
39 6.140434379631188e-06,
40 1.0502735676709563e-05,
41 5.883427093067439e-06,
42 6.942569598322734e-06,
43 1.1973874279647134e-05,
44 9.629780834075063e-06,
45 7.65456206863746e-06,
46 1.0055641723738518e-05
47 ],
48 "n": 8
49 }
50 },
51 "idea": {
52 "mean": 1.7339235796498542e-05,
53 "std": 4.534217009323715e-06,
54 "per_seed": [
55 1.389457884215517e-05,
56 2.2433650883613154e-05,
57 1.2446281289157923e-05,
58 2.6527255613473244e-05,
59 1.7356553144054487e-05,
60 1.4111134078120813e-05,
61 1.7410118744010106e-05,
62 1.453431377740344e-05
63 ],
64 "n": 8
65 },
66 "comparison": {
67 "delta_mean": 8.741357589769905e-06,
68 "idea_wins": 0,
69 "n_pairs": 8,
70 "per_seed_diffs": [
71 7.754144462523982e-06,
72 1.1930915206903592e-05,
73 6.562854196090484e-06,
74 1.958468601515051e-05,
75 5.382678864407353e-06,
76 4.481353244045749e-06,
77 9.755556675372645e-06,
78 4.478672053664923e-06
79 ],
80 "p_value": 0.0081,
81 "mde": 4.264723393974663e-06,
82 "mde_rel_pct": 49.60204473049084,
83 "verdict": "idea worse (significant)",
84 "system_worked": false,
85 "selected_idea_cfg": {
86 "lr": 0.006,
87 "weight_decay": 0.0
88 }
89 },
90 "mechanism_signature": {
91 "phase_chaining": {
92 "idea_by_seed": [
93 {
94 "seed": 0,
95 "high_jump_mse": 1.945050280482974e-05,
96 "low_jump_mse": 1.2042640264553484e-05
97 },
98 {
99 "seed": 1,
100 "high_jump_mse": 1.8415912563796155e-05,
101 "low_jump_mse": 2.3772785425535403e-05
102 },
103 {
104 "seed": 2,
105 "high_jump_mse": 1.0926943105005194e-05,
106 "low_jump_mse": 1.2952825272805057e-05
107 },
108 {
109 "seed": 3,
110 "high_jump_mse": 2.6379935661680065e-05,
111 "low_jump_mse": 2.6576328309602104e-05
112 },
113 {
114 "seed": 4,
115 "high_jump_mse": 1.1433454346843064e-05,
116 "low_jump_mse": 1.933102794282604e-05
117 },
118 {
119 "seed": 5,
120 "high_jump_mse": 1.3358734577195719e-05,
121 "low_jump_mse": 1.4361941794049926e-05
122 },
123 {
124 "seed": 6,
125 "high_jump_mse": 1.700813118077349e-05,
126 "low_jump_mse": 1.75441091414541e-05
127 },
128 {
129 "seed": 7,
130 "high_jump_mse": 1.2742822946165688e-05,
131 "low_jump_mse": 1.5131462532735895e-05
132 }
133 ],
134 "baseline_by_seed": [
135 {
136 "seed": 0,
137 "high_jump_mse": 9.345589205622673e-06,
138 "low_jump_mse": 5.072065960121108e-06
139 },
140 {
141 "seed": 1,
142 "high_jump_mse": 9.975614375434816e-06,
143 "low_jump_mse": 1.0678537364583462e-05
144 },
145 {
146 "seed": 2,
147 "high_jump_mse": 5.6180460887844674e-06,
148 "low_jump_mse": 5.971849532215856e-06
149 },
150 {
151 "seed": 3,
152 "high_jump_mse": 7.187292339949636e-06,
153 "low_jump_mse": 6.860935627628351e-06
154 },
155 {
156 "seed": 4,
157 "high_jump_mse": 1.1308992725389544e-05,
158 "low_jump_mse": 1.2195641829748638e-05
159 },
160 {
161 "seed": 5,
162 "high_jump_mse": 8.861595233611297e-06,
163 "low_jump_mse": 9.885759936878458e-06
164 },
165 {
166 "seed": 6,
167 "high_jump_mse": 8.615999831818044e-06,
168 "low_jump_mse": 7.334077508858172e-06
169 },
170 {
171 "seed": 7,
172 "high_jump_mse": 9.407116522197612e-06,
173 "low_jump_mse": 1.0271965038555209e-05
174 }
175 ],
176 "idea_high_jump_mse": 1.621455464828614e-05,
177 "baseline_high_jump_mse": 8.790030790351011e-06,
178 "idea_low_jump_mse": 1.771414008544525e-05,
179 "baseline_low_jump_mse": 8.533854099823657e-06,
180 "high_jump_delta_idea_minus_baseline": 7.424523857935128e-06,
181 "idea_params": 7137,
182 "baseline_params": 13313,
183 "confirmed": false,
184 "prediction": "phase chaining should reduce error on abrupt control changes"
185 },
186 "idea_cfg": {
187 "lr": 0.006,
188 "weight_decay": 0.0
189 },
190 "idea_grid": [
191 {
192 "cfg": {
193 "lr": 0.0015,
194 "weight_decay": 0.0
195 },
196 "result": {
197 "mean": 9.437663175049238e-05,
198 "std": 2.1191296338479005e-05,
199 "per_seed": [
200 0.00011513893696246669,
201 7.23859848221764e-05,
202 6.687596760457382e-05,
203 8.881815301720053e-05,
204 0.00011006097338395193,
205 7.692975486861542e-05,
206 0.00013098293857183307,
207 9.382034477312118e-05
208 ],
209 "n": 8
210 }
211 },
212 {
213 "cfg": {
214 "lr": 0.003,
215 "weight_decay": 0.0
216 },
217 "result": {
218 "mean": 3.437740951994783e-05,
219 "std": 8.875176143263352e-06,
220 "per_seed": [
221 3.745459980564192e-05,
222 3.1672490877099335e-05,
223 2.229700839961879e-05,
224 3.422219378990121e-05,
225 3.6611334508052096e-05,
226 2.5733035727171227e-05,
227 5.4019608796807006e-05,
228 3.3009004255291075e-05
229 ],
230 "n": 8
231 }
232 },
233 {
234 "cfg": {
235 "lr": 0.006,
236 "weight_decay": 0.0
237 },
238 "result": {
239 "mean": 1.7339235796498542e-05,
240 "std": 4.534217009323715e-06,
241 "per_seed": [
242 1.389457884215517e-05,
243 2.2433650883613154e-05,
244 1.2446281289157923e-05,
245 2.6527255613473244e-05,
246 1.7356553144054487e-05,
247 1.4111134078120813e-05,
248 1.7410118744010106e-05,
249 1.453431377740344e-05
250 ],
251 "n": 8
252 }
253 }
254 ],
255 "structural_match": "controlled pendulum rollout with an observed midpoint control event; primary metric is test MSE"
256 }
257}