Level-Adaptive Replay Memory / bench_report.json
Failed on benchmark
1{
2 "bench_version": 1,
3 "track": "tabular",
4 "model": "mlp_tiny",
5 "metric_direction": "lower is better",
6 "n_seeds": 8,
7 "baseline": {
8 "best_cfg": {
9 "lr": 0.01,
10 "depth": 8
11 },
12 "sweep": [
13 {
14 "cfg": {
15 "lr": 0.001,
16 "depth": 1
17 },
18 "mean": 217.19686126708984
19 },
20 {
21 "cfg": {
22 "lr": 0.001,
23 "depth": 2
24 },
25 "mean": 179.02721405029297
26 },
27 {
28 "cfg": {
29 "lr": 0.001,
30 "depth": 4
31 },
32 "mean": 25.813298225402832
33 },
34 {
35 "cfg": {
36 "lr": 0.001,
37 "depth": 8
38 },
39 "mean": 15.061450004577637
40 },
41 {
42 "cfg": {
43 "lr": 0.003,
44 "depth": 1
45 },
46 "mean": 142.7078514099121
47 },
48 {
49 "cfg": {
50 "lr": 0.003,
51 "depth": 2
52 },
53 "mean": 26.81587314605713
54 },
55 {
56 "cfg": {
57 "lr": 0.003,
58 "depth": 4
59 },
60 "mean": 13.492629766464233
61 },
62 {
63 "cfg": {
64 "lr": 0.003,
65 "depth": 8
66 },
67 "mean": 9.146221399307251
68 },
69 {
70 "cfg": {
71 "lr": 0.01,
72 "depth": 1
73 },
74 "mean": 35.42466497421265
75 },
76 {
77 "cfg": {
78 "lr": 0.01,
79 "depth": 2
80 },
81 "mean": 16.24430751800537
82 },
83 {
84 "cfg": {
85 "lr": 0.01,
86 "depth": 4
87 },
88 "mean": 8.620880007743835
89 },
90 {
91 "cfg": {
92 "lr": 0.01,
93 "depth": 8
94 },
95 "mean": 7.093326091766357
96 }
97 ],
98 "full": {
99 "mean": 7.334557235240936,
100 "std": 0.824177040857451,
101 "per_seed": [
102 6.754525661468506,
103 5.7614359855651855,
104 7.475412368774414,
105 8.381930351257324,
106 7.79284143447876,
107 6.628401756286621,
108 8.117091178894043,
109 7.764819145202637
110 ],
111 "n": 8
112 }
113 },
114 "idea": {
115 "mean": 23.1199893951416,
116 "std": 5.121285566952831,
117 "per_seed": [
118 15.503646850585938,
119 19.21010398864746,
120 18.322921752929688,
121 20.51085662841797,
122 24.305091857910156,
123 29.11318016052246,
124 29.386009216308594,
125 28.608104705810547
126 ],
127 "n": 8,
128 "selected_cfg": {
129 "lr": 0.01
130 }
131 },
132 "comparison": {
133 "delta_mean": 15.785432159900665,
134 "idea_wins": 0,
135 "n_pairs": 8,
136 "per_seed_diffs": [
137 8.749121189117432,
138 13.448668003082275,
139 10.847509384155273,
140 12.128926277160645,
141 16.512250423431396,
142 22.48477840423584,
143 21.26891803741455,
144 20.84328556060791
145 ],
146 "p_value": 0.0081,
147 "mde": 4.399418320126617,
148 "mde_rel_pct": 59.982057253413714,
149 "verdict": "idea worse (significant)",
150 "system_worked": false
151 },
152 "mechanism_signature": {
153 "math_sanity": {
154 "formula_check": [
155 {
156 "M": 1,
157 "predicted": 0.17300000000000004,
158 "observed": 0.17465833333333333,
159 "abs_error": 0.0016583333333332895
160 },
161 {
162 "M": 2,
163 "predicted": 0.3160710000000001,
164 "observed": 0.3166833333333333,
165 "abs_error": 0.0006123333333332148
166 },
167 {
168 "M": 4,
169 "predicted": 0.5322411229590001,
170 "observed": 0.5315916666666667,
171 "abs_error": 0.0006494562923333724
172 },
173 {
174 "M": 8,
175 "predicted": 0.7812016329493428,
176 "observed": 0.7828416666666667,
177 "abs_error": 0.0016400337173239077
178 },
179 {
180 "M": 16,
181 "predicted": 0.9521272745759659,
182 "observed": 0.9519,
183 "abs_error": 0.00022727457596594114
184 },
185 {
186 "M": 32,
187 "predicted": 0.9977082021604751,
188 "observed": 0.997875,
189 "abs_error": 0.00016679783952489036
190 }
191 ],
192 "max_abs_error": 0.0016583333333332895,
193 "stale_curve_optima": [
194 {
195 "stale": 0.0,
196 "optimal_depth": 16
197 },
198 {
199 "stale": 0.005,
200 "optimal_depth": 7
201 },
202 {
203 "stale": 0.015,
204 "optimal_depth": 4
205 },
206 {
207 "stale": 0.03,
208 "optimal_depth": 3
209 },
210 {
211 "stale": 0.05,
212 "optimal_depth": 3
213 }
214 ]
215 },
216 "trained_model_signature": {
217 "rho_hat": 0.13499999046325684,
218 "predicted_escape": 0.44015932468572094,
219 "observed_group_escape": 0.4399999976158142,
220 "absolute_error": 0.00015932706990673395,
221 "confirmed": true,
222 "mean_depth": 1.0,
223 "depth_trajectory": [
224 1,
225 1,
226 1,
227 1,
228 1,
229 1,
230 1,
231 1,
232 1,
233 1,
234 1,
235 1,
236 1,
237 1,
238 1,
239 1,
240 1,
241 1
242 ]
243 },
244 "idea_lr_trials": [
245 {
246 "cfg": {
247 "lr": 0.01
248 },
249 "result": {
250 "mean": 23.1199893951416,
251 "std": 5.121285566952831,
252 "per_seed": [
253 15.503646850585938,
254 19.21010398864746,
255 18.322921752929688,
256 20.51085662841797,
257 24.305091857910156,
258 29.11318016052246,
259 29.386009216308594,
260 28.608104705810547
261 ],
262 "n": 8,
263 "selected_cfg": {
264 "lr": 0.01
265 }
266 }
267 },
268 {
269 "cfg": {
270 "lr": 0.001
271 },
272 "result": {
273 "mean": 215.88136672973633,
274 "std": 10.926122378078592,
275 "per_seed": [
276 228.2001190185547,
277 218.22589111328125,
278 219.18821716308594,
279 203.1732177734375,
280 216.90838623046875,
281 202.43487548828125,
282 205.00531005859375,
283 233.9149169921875
284 ],
285 "n": 8
286 }
287 },
288 {
289 "cfg": {
290 "lr": 0.003
291 },
292 "result": {
293 "mean": 140.52062129974365,
294 "std": 14.768009273435721,
295 "per_seed": [
296 146.7877197265625,
297 148.9251708984375,
298 133.30699157714844,
299 141.8115234375,
300 114.57950592041016,
301 140.2008819580078,
302 129.85032653808594,
303 168.70285034179688
304 ],
305 "n": 8
306 }
307 }
308 ],
309 "track_choice": "tabular: replay-memory training intervention; same MLP and regression task in both arms"
310 }
311}