Connectivity-Preserving Wedge Token Pooling / bench_output.json
Mechanism confirmed, baseline not beaten
1{
2 "bench_version": 1,
3 "track": "wedge_graph_regression",
4 "model": "mlp_tiny",
5 "metric_direction": "lower is better",
6 "n_seeds": 8,
7 "baseline": {
8 "best_cfg": {
9 "lr": 0.003,
10 "epochs": 18
11 },
12 "sweep": [
13 {
14 "cfg": {
15 "lr": 0.001,
16 "epochs": 18
17 },
18 "mean": 0.061101581901311874
19 },
20 {
21 "cfg": {
22 "lr": 0.003,
23 "epochs": 18
24 },
25 "mean": 0.059752004221081734
26 },
27 {
28 "cfg": {
29 "lr": 0.01,
30 "epochs": 18
31 },
32 "mean": 0.07423717714846134
33 }
34 ],
35 "full": {
36 "mean": 0.059744860511273146,
37 "std": 0.007734674460330792,
38 "per_seed": [
39 0.05919400602579117,
40 0.052641332149505615,
41 0.07083674520254135,
42 0.0563359335064888,
43 0.06298892945051193,
44 0.04989643022418022,
45 0.05397595465183258,
46 0.0720895528793335
47 ],
48 "n": 8
49 }
50 },
51 "idea": {
52 "mean": 0.013510067714378238,
53 "std": 0.0028306420950107115,
54 "per_seed": [
55 0.011363038793206215,
56 0.012721219100058079,
57 0.012768835760653019,
58 0.012967556715011597,
59 0.009611863642930984,
60 0.013135809451341629,
61 0.019576922059059143,
62 0.015935296192765236
63 ],
64 "n": 8,
65 "signature": [
66 {
67 "pred_std": 0.16127879917621613,
68 "obs_std": 0.21961554884910583,
69 "pred_obs_corr": 0.887711709058279
70 },
71 {
72 "pred_std": 0.1682410091161728,
73 "obs_std": 0.21719904243946075,
74 "pred_obs_corr": 0.8598633105317619
75 },
76 {
77 "pred_std": 0.18861736357212067,
78 "obs_std": 0.23979876935482025,
79 "pred_obs_corr": 0.8878919008544431
80 },
81 {
82 "pred_std": 0.1867043375968933,
83 "obs_std": 0.22483287751674652,
84 "pred_obs_corr": 0.8631346192234685
85 },
86 {
87 "pred_std": 0.16417063772678375,
88 "obs_std": 0.21063706278800964,
89 "pred_obs_corr": 0.8946031027067823
90 },
91 {
92 "pred_std": 0.17319384217262268,
93 "obs_std": 0.22647514939308167,
94 "pred_obs_corr": 0.8694575455010861
95 },
96 {
97 "pred_std": 0.14724908769130707,
98 "obs_std": 0.20635201036930084,
99 "pred_obs_corr": 0.7448891050585023
100 },
101 {
102 "pred_std": 0.16360855102539062,
103 "obs_std": 0.22824695706367493,
104 "pred_obs_corr": 0.8425912593747041
105 }
106 ]
107 },
108 "comparison": {
109 "delta_mean": -0.04623479279689491,
110 "idea_wins": 8,
111 "n_pairs": 8,
112 "per_seed_diffs": [
113 -0.04783096723258495,
114 -0.039920113049447536,
115 -0.05806790944188833,
116 -0.0433683767914772,
117 -0.05337706580758095,
118 -0.03676062077283859,
119 -0.03439903259277344,
120 -0.05615425668656826
121 ],
122 "p_value": 0.0081,
123 "mde": 0.007544702982030243,
124 "mde_rel_pct": 12.628204196085866,
125 "verdict": "idea better (significant)",
126 "system_worked": true
127 },
128 "custom_track": {
129 "name": "wedge_graph_regression",
130 "file": "graph_wedge_bench.py",
131 "domain": "graph-nn"
132 },
133 "idea_grid": {
134 "0.001": {
135 "mean": 0.041265785694122314,
136 "std": 0.0038461557188544885,
137 "per_seed": [
138 0.03932115435600281,
139 0.03908920660614967,
140 0.03921026363968849,
141 0.04619516804814339,
142 0.03932721167802811,
143 0.03928273543715477,
144 0.038361623883247375,
145 0.049338921904563904
146 ],
147 "n": 8,
148 "signature": [
149 {
150 "pred_std": 0.10499881207942963,
151 "obs_std": 0.21961554884910583,
152 "pred_obs_corr": 0.4330130138308088
153 },
154 {
155 "pred_std": 0.08769117295742035,
156 "obs_std": 0.21719904243946075,
157 "pred_obs_corr": 0.43490863320131024
158 },
159 {
160 "pred_std": 0.0983906164765358,
161 "obs_std": 0.23979876935482025,
162 "pred_obs_corr": 0.5928418270204208
163 },
164 {
165 "pred_std": 0.09753165394067764,
166 "obs_std": 0.22483287751674652,
167 "pred_obs_corr": 0.31636030316148195
168 },
169 {
170 "pred_std": 0.11598880589008331,
171 "obs_std": 0.21063706278800964,
172 "pred_obs_corr": 0.3799591849447861
173 },
174 {
175 "pred_std": 0.09304504096508026,
176 "obs_std": 0.22647514939308167,
177 "pred_obs_corr": 0.49060817556425573
178 },
179 {
180 "pred_std": 0.10514671355485916,
181 "obs_std": 0.20635201036930084,
182 "pred_obs_corr": 0.35432711607503614
183 },
184 {
185 "pred_std": 0.08850710839033127,
186 "obs_std": 0.22824695706367493,
187 "pred_obs_corr": 0.2645536283175189
188 }
189 ]
190 },
191 "0.003": {
192 "mean": 0.024491973454132676,
193 "std": 0.0033225946932239044,
194 "per_seed": [
195 0.02352076955139637,
196 0.022262291982769966,
197 0.022041544318199158,
198 0.023201245814561844,
199 0.020999116823077202,
200 0.024190735071897507,
201 0.028309449553489685,
202 0.03141063451766968
203 ],
204 "n": 8,
205 "signature": [
206 {
207 "pred_std": 0.1270151138305664,
208 "obs_std": 0.21961554884910583,
209 "pred_obs_corr": 0.7369194685495051
210 },
211 {
212 "pred_std": 0.1252332478761673,
213 "obs_std": 0.21719904243946075,
214 "pred_obs_corr": 0.7475148343284106
215 },
216 {
217 "pred_std": 0.1388101875782013,
218 "obs_std": 0.23979876935482025,
219 "pred_obs_corr": 0.8221710532450354
220 },
221 {
222 "pred_std": 0.13323701918125153,
223 "obs_std": 0.22483287751674652,
224 "pred_obs_corr": 0.7602398729509038
225 },
226 {
227 "pred_std": 0.12740392982959747,
228 "obs_std": 0.21063706278800964,
229 "pred_obs_corr": 0.7434833188067616
230 },
231 {
232 "pred_std": 0.12677018344402313,
233 "obs_std": 0.22647514939308167,
234 "pred_obs_corr": 0.7601123761370298
235 },
236 {
237 "pred_std": 0.1245916560292244,
238 "obs_std": 0.20635201036930084,
239 "pred_obs_corr": 0.5823476350878336
240 },
241 {
242 "pred_std": 0.10668632388114929,
243 "obs_std": 0.22824695706367493,
244 "pred_obs_corr": 0.6635442880841551
245 }
246 ]
247 },
248 "0.01": {
249 "mean": 0.013510067714378238,
250 "std": 0.0028306420950107115,
251 "per_seed": [
252 0.011363038793206215,
253 0.012721219100058079,
254 0.012768835760653019,
255 0.012967556715011597,
256 0.009611863642930984,
257 0.013135809451341629,
258 0.019576922059059143,
259 0.015935296192765236
260 ],
261 "n": 8,
262 "signature": [
263 {
264 "pred_std": 0.16127879917621613,
265 "obs_std": 0.21961554884910583,
266 "pred_obs_corr": 0.887711709058279
267 },
268 {
269 "pred_std": 0.1682410091161728,
270 "obs_std": 0.21719904243946075,
271 "pred_obs_corr": 0.8598633105317619
272 },
273 {
274 "pred_std": 0.18861736357212067,
275 "obs_std": 0.23979876935482025,
276 "pred_obs_corr": 0.8878919008544431
277 },
278 {
279 "pred_std": 0.1867043375968933,
280 "obs_std": 0.22483287751674652,
281 "pred_obs_corr": 0.8631346192234685
282 },
283 {
284 "pred_std": 0.16417063772678375,
285 "obs_std": 0.21063706278800964,
286 "pred_obs_corr": 0.8946031027067823
287 },
288 {
289 "pred_std": 0.17319384217262268,
290 "obs_std": 0.22647514939308167,
291 "pred_obs_corr": 0.8694575455010861
292 },
293 {
294 "pred_std": 0.14724908769130707,
295 "obs_std": 0.20635201036930084,
296 "pred_obs_corr": 0.7448891050585023
297 },
298 {
299 "pred_std": 0.16360855102539062,
300 "obs_std": 0.22824695706367493,
301 "pred_obs_corr": 0.8425912593747041
302 }
303 ]
304 }
305 },
306 "mechanism_signature": {
307 "predicted": "graph-aware connected pooling should retain graph-smooth signal (higher prediction/observation correlation than random pooling)",
308 "observed": {
309 "baseline_best_corr_mean": 0.32037841882537543,
310 "idea_corr_mean": 0.8562678190386285
311 },
312 "confirmed": false
313 }
314}