Wide-tree invariant alignment layer / bench_report.json
Failed on benchmark
1{
2 "bench_version": 1,
3 "track": "unordered_pointset_denoising",
4 "model": "mlp_med",
5 "metric_direction": "lower is better",
6 "n_seeds": 8,
7 "baseline": {
8 "best_cfg": {
9 "lr": 0.006,
10 "weight_decay": 0.0
11 },
12 "sweep": [
13 {
14 "cfg": {
15 "lr": 0.001,
16 "weight_decay": 0.0
17 },
18 "mean": 0.04259627405554056
19 },
20 {
21 "cfg": {
22 "lr": 0.001,
23 "weight_decay": 0.0001
24 },
25 "mean": 0.04272811021655798
26 },
27 {
28 "cfg": {
29 "lr": 0.003,
30 "weight_decay": 0.0
31 },
32 "mean": 0.03448029048740864
33 },
34 {
35 "cfg": {
36 "lr": 0.003,
37 "weight_decay": 0.0001
38 },
39 "mean": 0.034148878417909145
40 },
41 {
42 "cfg": {
43 "lr": 0.006,
44 "weight_decay": 0.0
45 },
46 "mean": 0.03395335283130407
47 },
48 {
49 "cfg": {
50 "lr": 0.006,
51 "weight_decay": 0.0001
52 },
53 "mean": 0.034376708790659904
54 }
55 ],
56 "full": {
57 "mean": 0.034991197753697634,
58 "std": 0.002117625587373799,
59 "per_seed": [
60 0.036085259169340134,
61 0.03398258239030838,
62 0.03428776562213898,
63 0.0314578041434288,
64 0.034709829837083817,
65 0.03937392309308052,
66 0.035850487649440765,
67 0.034181930124759674
68 ],
69 "n": 8
70 }
71 },
72 "idea": {
73 "mean": 0.0419774423353374,
74 "std": 0.0049312628769484414,
75 "per_seed": [
76 0.039885085076093674,
77 0.03695981204509735,
78 0.038052886724472046,
79 0.04973628371953964,
80 0.04566482454538345,
81 0.036613475531339645,
82 0.04028519243001938,
83 0.04862197861075401
84 ],
85 "n": 8
86 },
87 "comparison": {
88 "delta_mean": 0.006986244581639767,
89 "idea_wins": 1,
90 "n_pairs": 8,
91 "per_seed_diffs": [
92 0.00379982590675354,
93 0.002977229654788971,
94 0.003765121102333069,
95 0.01827847957611084,
96 0.010954994708299637,
97 -0.0027604475617408752,
98 0.004434704780578613,
99 0.014440048485994339
100 ],
101 "p_value": 0.0163,
102 "mde": 0.005801091064441157,
103 "mde_rel_pct": 16.578715325136695,
104 "verdict": "idea worse (significant)",
105 "system_worked": false
106 },
107 "custom_track": {
108 "name": "unordered_pointset_denoising",
109 "file": "bench/custom_tracks/unordered_pointset_denoising.py",
110 "domain": "point-set-diffusion"
111 },
112 "baseline_grid": [
113 {
114 "lr": 0.001,
115 "weight_decay": 0.0
116 },
117 {
118 "lr": 0.001,
119 "weight_decay": 0.0001
120 },
121 {
122 "lr": 0.003,
123 "weight_decay": 0.0
124 },
125 {
126 "lr": 0.003,
127 "weight_decay": 0.0001
128 },
129 {
130 "lr": 0.006,
131 "weight_decay": 0.0
132 },
133 {
134 "lr": 0.006,
135 "weight_decay": 0.0001
136 }
137 ],
138 "idea_grid": [
139 {
140 "lr": 0.001,
141 "weight_decay": 0.0,
142 "tree_lambda": 0.02
143 },
144 {
145 "lr": 0.003,
146 "weight_decay": 0.0,
147 "tree_lambda": 0.05
148 },
149 {
150 "lr": 0.006,
151 "weight_decay": 0.0001,
152 "tree_lambda": 0.1
153 }
154 ],
155 "idea_trials": [
156 {
157 "cfg": {
158 "lr": 0.001,
159 "weight_decay": 0.0,
160 "tree_lambda": 0.02
161 },
162 "mean": 0.04521314054727554
163 },
164 {
165 "cfg": {
166 "lr": 0.003,
167 "weight_decay": 0.0,
168 "tree_lambda": 0.05
169 },
170 "mean": 0.04115851689130068
171 },
172 {
173 "cfg": {
174 "lr": 0.006,
175 "weight_decay": 0.0001,
176 "tree_lambda": 0.1
177 },
178 "mean": 0.04825861472636461
179 }
180 ],
181 "math_check": {
182 "gram_tree_orthogonal_invariance_relative_error": 2.9398856327134126e-07
183 },
184 "mechanism_signature": {
185 "claim": "tree aggregation preserves target set correspondence",
186 "predicted": "tree-feature correlation is higher for the idea system",
187 "observed_baseline_mean_tree_corr": 0.9266873237022839,
188 "observed_idea_mean_tree_corr": 0.9220414828405017,
189 "per_seed": [
190 {
191 "seed": 0,
192 "baseline_tree_corr": 0.9256211589939427,
193 "idea_tree_corr": 0.917472849753899,
194 "baseline_mse": 0.036085259169340134,
195 "idea_mse": 0.039885085076093674
196 },
197 {
198 "seed": 1,
199 "baseline_tree_corr": 0.9170005706547195,
200 "idea_tree_corr": 0.924394805959731,
201 "baseline_mse": 0.03398258239030838,
202 "idea_mse": 0.03695981204509735
203 },
204 {
205 "seed": 2,
206 "baseline_tree_corr": 0.9286817916553544,
207 "idea_tree_corr": 0.9288227539982717,
208 "baseline_mse": 0.03428776562213898,
209 "idea_mse": 0.038052886724472046
210 },
211 {
212 "seed": 3,
213 "baseline_tree_corr": 0.9419077791531315,
214 "idea_tree_corr": 0.9076383879566745,
215 "baseline_mse": 0.0314578041434288,
216 "idea_mse": 0.04973628371953964
217 },
218 {
219 "seed": 4,
220 "baseline_tree_corr": 0.9288199555878167,
221 "idea_tree_corr": 0.922377819116332,
222 "baseline_mse": 0.034709829837083817,
223 "idea_mse": 0.04566482454538345
224 },
225 {
226 "seed": 5,
227 "baseline_tree_corr": 0.9205480277200384,
228 "idea_tree_corr": 0.9233275703070802,
229 "baseline_mse": 0.03937392309308052,
230 "idea_mse": 0.036613475531339645
231 },
232 {
233 "seed": 6,
234 "baseline_tree_corr": 0.9299053733445586,
235 "idea_tree_corr": 0.9330525084728803,
236 "baseline_mse": 0.035850487649440765,
237 "idea_mse": 0.04028519243001938
238 },
239 {
240 "seed": 7,
241 "baseline_tree_corr": 0.9210139325087091,
242 "idea_tree_corr": 0.9192451671591447,
243 "baseline_mse": 0.034181930124759674,
244 "idea_mse": 0.04862197861075401
245 }
246 ],
247 "confirmed": false
248 }
249}