Intrinsic-Rank Filter Memory for Actor-Critic / bench_report.json
Mechanism confirmed, baseline not beaten
1{
2 "bench_version": 1,
3 "track": "dynamics",
4 "model": "rnn_small",
5 "metric_direction": "lower is better",
6 "n_seeds": 8,
7 "baseline": {
8 "best_cfg": {
9 "lr": 0.01,
10 "tau": 0.01
11 },
12 "sweep": [
13 {
14 "cfg": {
15 "lr": 0.001,
16 "tau": 0.01
17 },
18 "mean": 0.0034653106704354286
19 },
20 {
21 "cfg": {
22 "lr": 0.001,
23 "tau": 0.001
24 },
25 "mean": 0.0034653106704354286
26 },
27 {
28 "cfg": {
29 "lr": 0.001,
30 "tau": 0.0001
31 },
32 "mean": 0.0034653106704354286
33 },
34 {
35 "cfg": {
36 "lr": 0.003,
37 "tau": 0.01
38 },
39 "mean": 0.002506730961613357
40 },
41 {
42 "cfg": {
43 "lr": 0.003,
44 "tau": 0.001
45 },
46 "mean": 0.002506730961613357
47 },
48 {
49 "cfg": {
50 "lr": 0.003,
51 "tau": 0.0001
52 },
53 "mean": 0.002506730961613357
54 },
55 {
56 "cfg": {
57 "lr": 0.01,
58 "tau": 0.01
59 },
60 "mean": 0.0010691375500755385
61 },
62 {
63 "cfg": {
64 "lr": 0.01,
65 "tau": 0.001
66 },
67 "mean": 0.0010691375500755385
68 },
69 {
70 "cfg": {
71 "lr": 0.01,
72 "tau": 0.0001
73 },
74 "mean": 0.0010691375500755385
75 }
76 ],
77 "full": {
78 "mean": 0.0011299664110993035,
79 "std": 0.00033636016341449245,
80 "per_seed": [
81 0.0008199093863368034,
82 0.0008286746451631188,
83 0.0016839810414239764,
84 0.0009439851273782551,
85 0.001039800001308322,
86 0.0017109999898821115,
87 0.0009939224692061543,
88 0.0010184586280956864
89 ],
90 "n": 8
91 }
92 },
93 "idea": {
94 "mean": 0.0010912194484262727,
95 "std": 0.00044965589408538725,
96 "per_seed": [
97 0.0005372872692532837,
98 0.0008794514578767121,
99 0.001270528300665319,
100 0.0019876640290021896,
101 0.0010541900992393494,
102 0.0015042235609143972,
103 0.0008550031343474984,
104 0.0006414077361114323
105 ],
106 "n": 8
107 },
108 "comparison": {
109 "delta_mean": -3.8746962673030794e-05,
110 "idea_wins": 5,
111 "n_pairs": 8,
112 "per_seed_diffs": [
113 -0.0002826221170835197,
114 5.0776812713593245e-05,
115 -0.00041345274075865746,
116 0.0010436789016239345,
117 1.4390097931027412e-05,
118 -0.0002067764289677143,
119 -0.00013891933485865593,
120 -0.0003770508919842541
121 ],
122 "p_value": 0.8938,
123 "mde": 0.0003915640556028904,
124 "mde_rel_pct": 34.65271637782152,
125 "verdict": "no significant win",
126 "system_worked": false
127 },
128 "mechanism_signature": {
129 "idea_config": {
130 "lr": 0.01,
131 "tau": 0.01
132 },
133 "idea_sweep": [
134 {
135 "cfg": {
136 "lr": 0.001,
137 "tau": 0.01
138 },
139 "result": {
140 "mean": 0.004125978332012892,
141 "std": 0.0021316471939702973,
142 "per_seed": [
143 0.0034050061367452145,
144 0.0024806063156574965,
145 0.0038227855693548918,
146 0.009371915832161903,
147 0.004681112710386515,
148 0.002079644240438938,
149 0.004045997746288776,
150 0.003120758105069399
151 ],
152 "n": 8
153 }
154 },
155 {
156 "cfg": {
157 "lr": 0.001,
158 "tau": 0.001
159 },
160 "result": {
161 "mean": 0.004125978332012892,
162 "std": 0.0021316471939702973,
163 "per_seed": [
164 0.0034050061367452145,
165 0.0024806063156574965,
166 0.0038227855693548918,
167 0.009371915832161903,
168 0.004681112710386515,
169 0.002079644240438938,
170 0.004045997746288776,
171 0.003120758105069399
172 ],
173 "n": 8
174 }
175 },
176 {
177 "cfg": {
178 "lr": 0.001,
179 "tau": 0.0001
180 },
181 "result": {
182 "mean": 0.004125978332012892,
183 "std": 0.0021316471939702973,
184 "per_seed": [
185 0.0034050061367452145,
186 0.0024806063156574965,
187 0.0038227855693548918,
188 0.009371915832161903,
189 0.004681112710386515,
190 0.002079644240438938,
191 0.004045997746288776,
192 0.003120758105069399
193 ],
194 "n": 8
195 }
196 },
197 {
198 "cfg": {
199 "lr": 0.003,
200 "tau": 0.01
201 },
202 "result": {
203 "mean": 0.002102683240082115,
204 "std": 0.000509014269608427,
205 "per_seed": [
206 0.0013083082158118486,
207 0.002864542882889509,
208 0.0026755419094115496,
209 0.0018761787796393037,
210 0.0020888173021376133,
211 0.0016800755402073264,
212 0.002548397518694401,
213 0.0017796037718653679
214 ],
215 "n": 8
216 }
217 },
218 {
219 "cfg": {
220 "lr": 0.003,
221 "tau": 0.001
222 },
223 "result": {
224 "mean": 0.002102683240082115,
225 "std": 0.000509014269608427,
226 "per_seed": [
227 0.0013083082158118486,
228 0.002864542882889509,
229 0.0026755419094115496,
230 0.0018761787796393037,
231 0.0020888173021376133,
232 0.0016800755402073264,
233 0.002548397518694401,
234 0.0017796037718653679
235 ],
236 "n": 8
237 }
238 },
239 {
240 "cfg": {
241 "lr": 0.003,
242 "tau": 0.0001
243 },
244 "result": {
245 "mean": 0.002102683240082115,
246 "std": 0.000509014269608427,
247 "per_seed": [
248 0.0013083082158118486,
249 0.002864542882889509,
250 0.0026755419094115496,
251 0.0018761787796393037,
252 0.0020888173021376133,
253 0.0016800755402073264,
254 0.002548397518694401,
255 0.0017796037718653679
256 ],
257 "n": 8
258 }
259 },
260 {
261 "cfg": {
262 "lr": 0.01,
263 "tau": 0.01
264 },
265 "result": {
266 "mean": 0.0010912194484262727,
267 "std": 0.00044965589408538725,
268 "per_seed": [
269 0.0005372872692532837,
270 0.0008794514578767121,
271 0.001270528300665319,
272 0.0019876640290021896,
273 0.0010541900992393494,
274 0.0015042235609143972,
275 0.0008550031343474984,
276 0.0006414077361114323
277 ],
278 "n": 8
279 }
280 },
281 {
282 "cfg": {
283 "lr": 0.01,
284 "tau": 0.001
285 },
286 "result": {
287 "mean": 0.0010912194484262727,
288 "std": 0.00044965589408538725,
289 "per_seed": [
290 0.0005372872692532837,
291 0.0008794514578767121,
292 0.001270528300665319,
293 0.0019876640290021896,
294 0.0010541900992393494,
295 0.0015042235609143972,
296 0.0008550031343474984,
297 0.0006414077361114323
298 ],
299 "n": 8
300 }
301 },
302 {
303 "cfg": {
304 "lr": 0.01,
305 "tau": 0.0001
306 },
307 "result": {
308 "mean": 0.0010912194484262727,
309 "std": 0.00044965589408538725,
310 "per_seed": [
311 0.0005372872692532837,
312 0.0008794514578767121,
313 0.001270528300665319,
314 0.0019876640290021896,
315 0.0010541900992393494,
316 0.0015042235609143972,
317 0.0008550031343474984,
318 0.0006414077361114323
319 ],
320 "n": 8
321 }
322 }
323 ],
324 "mechanism_signature": {
325 "prediction": "causal lifted ambient width 9 has deterministic duplicate directions and SVD rank 6",
326 "predicted_intrinsic_rank": 6,
327 "observed_rank_tau_1e-3": 6,
328 "ambient_coordinate_width": 9,
329 "trained_model_behavior": {
330 "baseline": {
331 "metric": 0.00200747512280941,
332 "prediction_rms_under_input_perturbation": 0.0009409512858837843
333 },
334 "idea": {
335 "metric": 0.0012641767971217632,
336 "prediction_rms_under_input_perturbation": 0.0011408096179366112
337 }
338 },
339 "confirmed": true
340 }
341 }
342}