Invariant-domain learned reconstruction / bench_report.json
Mechanism confirmed, baseline not beaten
1{
2 "bench_version": 1,
3 "track": "dynamics",
4 "model": "rnn_small",
5 "metric_direction": "lower is better",
6 "n_seeds": 8,
7 "baseline": {
8 "best_cfg": {
9 "lr": 0.01
10 },
11 "sweep": [
12 {
13 "cfg": {
14 "lr": 0.001
15 },
16 "mean": 0.0015202847716864198
17 },
18 {
19 "cfg": {
20 "lr": 0.003
21 },
22 "mean": 0.0009046953928191215
23 },
24 {
25 "cfg": {
26 "lr": 0.01
27 },
28 "mean": 0.00043617923438432626
29 }
30 ],
31 "full": {
32 "mean": 0.0003015457559740753,
33 "std": 0.00019741488644742065,
34 "per_seed": [
35 0.0006247494602575898,
36 0.0004462092474568635,
37 0.00017418812785763294,
38 0.0004995701019652188,
39 0.0003730663738679141,
40 0.00010612085316097364,
41 6.512444815598428e-05,
42 0.00012333743507042527
43 ],
44 "n": 8
45 }
46 },
47 "idea": {
48 "mean": 0.0002918587179010501,
49 "std": 0.00018751789044513985,
50 "per_seed": [
51 0.0005810419097542763,
52 0.0004426969971973449,
53 0.00017546206072438508,
54 0.0005002013058401644,
55 0.00034365191822871566,
56 0.00010483426012797281,
57 6.512441905215383e-05,
58 0.00012185687228338793
59 ],
60 "n": 8,
61 "sweep": [
62 {
63 "cfg": {
64 "lr": 0.001,
65 "floor": 1e-05
66 },
67 "mean": 0.0015202843060251325,
68 "sweep": {
69 "mean": 0.0015202843060251325,
70 "std": 0.0002737330879148898,
71 "per_seed": [
72 0.001321663847193122,
73 0.001269076718017459,
74 0.0015261193038895726,
75 0.0019642773550003767
76 ],
77 "n": 4
78 }
79 },
80 {
81 "cfg": {
82 "lr": 0.003,
83 "floor": 1e-05
84 },
85 "mean": 0.0009028831991599873,
86 "sweep": {
87 "mean": 0.0009028831991599873,
88 "std": 0.00015086129898563282,
89 "per_seed": [
90 0.0009533172706142068,
91 0.0007026336970739067,
92 0.0008414627518504858,
93 0.0011141190771013498
94 ],
95 "n": 4
96 }
97 },
98 {
99 "cfg": {
100 "lr": 0.01,
101 "floor": 1e-05
102 },
103 "mean": 0.00042485056837904267,
104 "sweep": {
105 "mean": 0.00042485056837904267,
106 "std": 0.0001521402244052453,
107 "per_seed": [
108 0.0005810419097542763,
109 0.0004426969971973449,
110 0.00017546206072438508,
111 0.0005002013058401644
112 ],
113 "n": 4
114 }
115 }
116 ]
117 },
118 "comparison": {
119 "delta_mean": -9.687038073025178e-06,
120 "idea_wins": 6,
121 "n_pairs": 8,
122 "per_seed_diffs": [
123 -4.370755050331354e-05,
124 -3.5122502595186234e-06,
125 1.2739328667521477e-06,
126 6.312038749456406e-07,
127 -2.9414455639198422e-05,
128 -1.2865930330008268e-06,
129 -2.9103830456733704e-11,
130 -1.4805627870373428e-06
131 ],
132 "p_value": 0.0823,
133 "mde": 1.4284899018237163e-05,
134 "mde_rel_pct": 4.737224363212485,
135 "verdict": "no significant win",
136 "system_worked": false
137 },
138 "mechanism_signature": {
139 "track_choice": "dynamics matches stability/control structure",
140 "idea_sweep": [
141 {
142 "cfg": {
143 "lr": 0.001,
144 "floor": 1e-05
145 },
146 "mean": 0.0015202843060251325,
147 "sweep": {
148 "mean": 0.0015202843060251325,
149 "std": 0.0002737330879148898,
150 "per_seed": [
151 0.001321663847193122,
152 0.001269076718017459,
153 0.0015261193038895726,
154 0.0019642773550003767
155 ],
156 "n": 4
157 }
158 },
159 {
160 "cfg": {
161 "lr": 0.003,
162 "floor": 1e-05
163 },
164 "mean": 0.0009028831991599873,
165 "sweep": {
166 "mean": 0.0009028831991599873,
167 "std": 0.00015086129898563282,
168 "per_seed": [
169 0.0009533172706142068,
170 0.0007026336970739067,
171 0.0008414627518504858,
172 0.0011141190771013498
173 ],
174 "n": 4
175 }
176 },
177 {
178 "cfg": {
179 "lr": 0.01,
180 "floor": 1e-05
181 },
182 "mean": 0.00042485056837904267,
183 "sweep": {
184 "mean": 0.00042485056837904267,
185 "std": 0.0001521402244052453,
186 "per_seed": [
187 0.0005810419097542763,
188 0.0004426969971973449,
189 0.00017546206072438508,
190 0.0005002013058401644
191 ],
192 "n": 4
193 }
194 }
195 ],
196 "mechanism_signature": {
197 "prediction": "hard convex interpolation eliminates out-of-domain reconstructed states",
198 "observed": [
199 {
200 "seed": 0,
201 "raw_invalid_rate": 0.0,
202 "limited_invalid_rate": 0.0,
203 "activation_rate": 0.0,
204 "raw_anchor_displacement": 0.03173816576600075,
205 "limited_anchor_displacement": 0.03173816576600075
206 },
207 {
208 "seed": 1,
209 "raw_invalid_rate": 0.0,
210 "limited_invalid_rate": 0.0,
211 "activation_rate": 0.0,
212 "raw_anchor_displacement": 0.03535204008221626,
213 "limited_anchor_displacement": 0.03535204008221626
214 },
215 {
216 "seed": 2,
217 "raw_invalid_rate": 0.0,
218 "limited_invalid_rate": 0.0,
219 "activation_rate": 0.0,
220 "raw_anchor_displacement": 0.03403977304697037,
221 "limited_anchor_displacement": 0.03403977304697037
222 },
223 {
224 "seed": 3,
225 "raw_invalid_rate": 0.0,
226 "limited_invalid_rate": 0.0,
227 "activation_rate": 0.0,
228 "raw_anchor_displacement": 0.03182002902030945,
229 "limited_anchor_displacement": 0.03182002902030945
230 },
231 {
232 "seed": 4,
233 "raw_invalid_rate": 0.0,
234 "limited_invalid_rate": 0.0,
235 "activation_rate": 0.0,
236 "raw_anchor_displacement": 0.03627397492527962,
237 "limited_anchor_displacement": 0.03627397492527962
238 },
239 {
240 "seed": 5,
241 "raw_invalid_rate": 0.0,
242 "limited_invalid_rate": 0.0,
243 "activation_rate": 0.0,
244 "raw_anchor_displacement": 0.0337686762213707,
245 "limited_anchor_displacement": 0.0337686762213707
246 },
247 {
248 "seed": 6,
249 "raw_invalid_rate": 0.0,
250 "limited_invalid_rate": 0.0,
251 "activation_rate": 0.0,
252 "raw_anchor_displacement": 0.030422382056713104,
253 "limited_anchor_displacement": 0.030422382056713104
254 },
255 {
256 "seed": 7,
257 "raw_invalid_rate": 0.0,
258 "limited_invalid_rate": 0.0,
259 "activation_rate": 0.0,
260 "raw_anchor_displacement": 0.0314624160528183,
261 "limited_anchor_displacement": 0.0314624160528183
262 }
263 ],
264 "raw_invalid_rate_mean": 0.0,
265 "limited_invalid_rate_mean": 0.0,
266 "confirmed": true
267 }
268 }
269}