Continuation Maps for Training-Mode Transitions / bench_report.json
Mechanism confirmed, baseline not beaten
1{
2 "bench_version": 1,
3 "track": "dynamics",
4 "model": "rnn_small",
5 "metric_direction": "lower is better",
6 "n_seeds": 8,
7 "baseline": {
8 "best_cfg": {
9 "lr": 0.01
10 },
11 "sweep": [
12 {
13 "cfg": {
14 "lr": 0.001
15 },
16 "mean": 0.5504409596323967
17 },
18 {
19 "cfg": {
20 "lr": 0.003
21 },
22 "mean": 0.2672337554395199
23 },
24 {
25 "cfg": {
26 "lr": 0.01
27 },
28 "mean": 0.0029119684331817552
29 }
30 ],
31 "full": {
32 "mean": 0.0029119684331817552,
33 "std": 0.0019628704033910396,
34 "per_seed": [
35 0.0013499484630301595,
36 0.002197778085246682,
37 0.0036032795906066895,
38 0.00771956006065011,
39 0.0013576406054198742,
40 0.0016320815775543451,
41 0.002993914531543851,
42 0.0024415445514023304
43 ],
44 "n": 8
45 }
46 },
47 "idea": {
48 "mean": 0.2629458736628294,
49 "std": 0.06242340567924847,
50 "per_seed": [
51 0.3094339966773987,
52 0.3548787236213684,
53 0.20706547796726227,
54 0.3201036751270294,
55 0.2438417375087738,
56 0.22770699858665466,
57 0.15347930788993835,
58 0.2870570719242096
59 ],
60 "n": 8,
61 "selected_cfg": {
62 "lr": 0.01
63 },
64 "trials": [
65 {
66 "cfg": {
67 "lr": 0.001
68 },
69 "mean": 0.6568634137511253,
70 "std": 0.09002834001247538,
71 "per_seed": [
72 0.5851488709449768,
73 0.6946209669113159,
74 0.6762194633483887,
75 0.8284632563591003,
76 0.5672825574874878,
77 0.6657136678695679,
78 0.525349497795105,
79 0.7121090292930603
80 ]
81 },
82 {
83 "cfg": {
84 "lr": 0.003
85 },
86 "mean": 0.527643658220768,
87 "std": 0.12691734560746284,
88 "per_seed": [
89 0.49636369943618774,
90 0.5941106081008911,
91 0.561820387840271,
92 0.7420464754104614,
93 0.4776306748390198,
94 0.4868730902671814,
95 0.26752692461013794,
96 0.5947774052619934
97 ]
98 },
99 {
100 "cfg": {
101 "lr": 0.01
102 },
103 "mean": 0.2629458736628294,
104 "std": 0.06242340567924847,
105 "per_seed": [
106 0.3094339966773987,
107 0.3548787236213684,
108 0.20706547796726227,
109 0.3201036751270294,
110 0.2438417375087738,
111 0.22770699858665466,
112 0.15347930788993835,
113 0.2870570719242096
114 ]
115 }
116 ]
117 },
118 "comparison": {
119 "delta_mean": 0.26003390522964764,
120 "idea_wins": 0,
121 "n_pairs": 8,
122 "per_seed_diffs": [
123 0.3080840482143685,
124 0.3526809455361217,
125 0.20346219837665558,
126 0.3123841150663793,
127 0.24248409690335393,
128 0.22607491700910032,
129 0.1504853933583945,
130 0.28461552737280726
131 ],
132 "p_value": 0.0081,
133 "mde": 0.05555371377308973,
134 "mde_rel_pct": 1907.771840520575,
135 "verdict": "idea worse (significant)",
136 "system_worked": false
137 },
138 "mechanism_signature": {
139 "prediction": "continuation should reduce late-window loss oscillation while remaining on stable side",
140 "baseline_feature_mean": 0.583179637221284,
141 "idea_feature_mean": 0.16445192880727005,
142 "relative_feature_reduction": 0.7180081088023521,
143 "observed_final_lr_mean": 0.00029296875000000004,
144 "n_trained_models": 16,
145 "confirmed": true
146 },
147 "bench_report": {
148 "track_match": "stability/control -> dynamics",
149 "paired_seeds": [
150 0,
151 1,
152 2,
153 3,
154 4,
155 5,
156 6,
157 7
158 ],
159 "baseline_grid": [
160 {
161 "lr": 0.001
162 },
163 {
164 "lr": 0.003
165 },
166 {
167 "lr": 0.01
168 }
169 ],
170 "idea_grid": [
171 {
172 "lr": 0.001
173 },
174 {
175 "lr": 0.003
176 },
177 {
178 "lr": 0.01
179 }
180 ],
181 "weight_decay": 0.0001,
182 "epochs": 18,
183 "batch_size": 64,
184 "signature_rows": [
185 {
186 "seed": 0,
187 "baseline_F_osc": 0.6125851979328734,
188 "idea_F_osc": 0.13701609105038687,
189 "idea_final_lr": 0.000625,
190 "baseline_test_mse": 0.0013499484630301595,
191 "idea_test_mse": 0.3094339966773987
192 },
193 {
194 "seed": 1,
195 "baseline_F_osc": 0.6589754319853819,
196 "idea_F_osc": 0.148168541527418,
197 "idea_final_lr": 7.8125e-05,
198 "baseline_test_mse": 0.002197778085246682,
199 "idea_test_mse": 0.3548787236213684
200 },
201 {
202 "seed": 2,
203 "baseline_F_osc": 0.4906425914915999,
204 "idea_F_osc": 0.15470220077295294,
205 "idea_final_lr": 0.0003125,
206 "baseline_test_mse": 0.0036032795906066895,
207 "idea_test_mse": 0.20706547796726227
208 },
209 {
210 "seed": 3,
211 "baseline_F_osc": 0.5542223686616028,
212 "idea_F_osc": 0.1682112135875921,
213 "idea_final_lr": 3.90625e-05,
214 "baseline_test_mse": 0.00771956006065011,
215 "idea_test_mse": 0.3201036751270294
216 },
217 {
218 "seed": 4,
219 "baseline_F_osc": 0.5676767941860419,
220 "idea_F_osc": 0.17183414086022114,
221 "idea_final_lr": 1.953125e-05,
222 "baseline_test_mse": 0.0013576406054198742,
223 "idea_test_mse": 0.2438417375087738
224 },
225 {
226 "seed": 5,
227 "baseline_F_osc": 0.6356397616859335,
228 "idea_F_osc": 0.14964675665547397,
229 "idea_final_lr": 0.000625,
230 "baseline_test_mse": 0.0016320815775543451,
231 "idea_test_mse": 0.22770699858665466
232 },
233 {
234 "seed": 6,
235 "baseline_F_osc": 0.480638587229412,
236 "idea_F_osc": 0.18274592980114265,
237 "idea_final_lr": 0.000625,
238 "baseline_test_mse": 0.002993914531543851,
239 "idea_test_mse": 0.15347930788993835
240 },
241 {
242 "seed": 7,
243 "baseline_F_osc": 0.6650563645974263,
244 "idea_F_osc": 0.20329055620297268,
245 "idea_final_lr": 1.953125e-05,
246 "baseline_test_mse": 0.0024415445514023304,
247 "idea_test_mse": 0.2870570719242096
248 }
249 ]
250 }
251}