Resolving Landmark Bottleneck / bench_report.json
Beats tuned baseline
1{
2 "bench_version": 1,
3 "track": "graph_landmark_forecast",
4 "model": "transformer_tiny",
5 "metric_direction": "lower is better",
6 "n_seeds": 8,
7 "baseline": {
8 "best_cfg": {
9 "lr": 0.003
10 },
11 "sweep": [
12 {
13 "cfg": {
14 "lr": 0.001
15 },
16 "mean": 0.09760121069848537
17 },
18 {
19 "cfg": {
20 "lr": 0.003
21 },
22 "mean": 0.09715960919857025
23 },
24 {
25 "cfg": {
26 "lr": 0.01
27 },
28 "mean": 0.09787633270025253
29 }
30 ],
31 "full": {
32 "mean": 0.09728806372731924,
33 "std": 0.0028138072522287676,
34 "per_seed": [
35 0.0970306470990181,
36 0.09655177593231201,
37 0.09898804873228073,
38 0.09606796503067017,
39 0.10214304178953171,
40 0.09961570054292679,
41 0.09206581115722656,
42 0.09584151953458786
43 ],
44 "n": 8
45 },
46 "full_grid": [
47 {
48 "cfg": {
49 "lr": 0.001
50 },
51 "result": {
52 "mean": 0.09813422616571188,
53 "std": 0.003308826432344093,
54 "per_seed": [
55 0.09822424501180649,
56 0.09727376699447632,
57 0.0995187908411026,
58 0.09538803994655609,
59 0.10483681410551071,
60 0.10019651055335999,
61 0.09305645525455475,
62 0.0965791866183281
63 ],
64 "n": 8
65 }
66 },
67 {
68 "cfg": {
69 "lr": 0.003
70 },
71 "result": {
72 "mean": 0.09728806372731924,
73 "std": 0.0028138072522287676,
74 "per_seed": [
75 0.0970306470990181,
76 0.09655177593231201,
77 0.09898804873228073,
78 0.09606796503067017,
79 0.10214304178953171,
80 0.09961570054292679,
81 0.09206581115722656,
82 0.09584151953458786
83 ],
84 "n": 8
85 }
86 },
87 {
88 "cfg": {
89 "lr": 0.01
90 },
91 "result": {
92 "mean": 0.09821864496916533,
93 "std": 0.002263958786368283,
94 "per_seed": [
95 0.09782364219427109,
96 0.09844150394201279,
97 0.09902690351009369,
98 0.09621328115463257,
99 0.10131707042455673,
100 0.10156738013029099,
101 0.09458042681217194,
102 0.09677895158529282
103 ],
104 "n": 8
105 }
106 }
107 ]
108 },
109 "idea": {
110 "mean": 0.06668837741017342,
111 "std": 0.00810202264282473,
112 "per_seed": [
113 0.06714659929275513,
114 0.07387638092041016,
115 0.06558588147163391,
116 0.059248507022857666,
117 0.05166328698396683,
118 0.06404716521501541,
119 0.07355235517024994,
120 0.07838684320449829
121 ],
122 "n": 8
123 },
124 "comparison": {
125 "delta_mean": -0.030599686317145824,
126 "idea_wins": 8,
127 "n_pairs": 8,
128 "per_seed_diffs": [
129 -0.02988404780626297,
130 -0.022675395011901855,
131 -0.03340216726064682,
132 -0.0368194580078125,
133 -0.05047975480556488,
134 -0.03556853532791138,
135 -0.018513455986976624,
136 -0.01745467633008957
137 ],
138 "p_value": 0.0081,
139 "mde": 0.00920648763417379,
140 "mde_rel_pct": 9.463121457507778,
141 "verdict": "idea better (significant)",
142 "system_worked": true
143 },
144 "custom_track": {
145 "name": "graph_landmark_forecast",
146 "file": "graph_landmark_track.py",
147 "domain": "graph-nn"
148 },
149 "idea_config": {
150 "lr": 0.01
151 },
152 "idea_sweep": [
153 {
154 "cfg": {
155 "lr": 0.001
156 },
157 "result": {
158 "mean": 0.0952951293438673,
159 "std": 0.0034853121866759325,
160 "per_seed": [
161 0.0953633040189743,
162 0.09373979270458221,
163 0.09905679523944855,
164 0.092075414955616,
165 0.10213538259267807,
166 0.0953550785779953,
167 0.09400807321071625,
168 0.09062719345092773
169 ],
170 "n": 8
171 }
172 },
173 {
174 "cfg": {
175 "lr": 0.003
176 },
177 "result": {
178 "mean": 0.08661147300153971,
179 "std": 0.004209441400448157,
180 "per_seed": [
181 0.08820009231567383,
182 0.09083075076341629,
183 0.0886211097240448,
184 0.08095572143793106,
185 0.09157451242208481,
186 0.07902113348245621,
187 0.0880790650844574,
188 0.08560939878225327
189 ],
190 "n": 8
191 }
192 },
193 {
194 "cfg": {
195 "lr": 0.01
196 },
197 "result": {
198 "mean": 0.06668837741017342,
199 "std": 0.00810202264282473,
200 "per_seed": [
201 0.06714659929275513,
202 0.07387638092041016,
203 0.06558588147163391,
204 0.059248507022857666,
205 0.05166328698396683,
206 0.06404716521501541,
207 0.07355235517024994,
208 0.07838684320449829
209 ],
210 "n": 8
211 }
212 }
213 ],
214 "mechanism_signature": {
215 "prediction": "expected collisions are bounded by the hypergeometric union bound",
216 "c_min": 1,
217 "s": 8,
218 "predicted_collision_upper_bound": 114.0,
219 "observed_idea_collision_count": 0.0,
220 "baseline_collision_count": 19.0,
221 "trained_model_observations": [
222 {
223 "seed": 0,
224 "baseline_metric": 0.0970306470990181,
225 "idea_metric": 0.08820009231567383,
226 "baseline_collisions": 19,
227 "idea_collisions": 0,
228 "idea_pred_target_corr": 0.892614061575404
229 },
230 {
231 "seed": 1,
232 "baseline_metric": 0.09655177593231201,
233 "idea_metric": 0.09083075076341629,
234 "baseline_collisions": 19,
235 "idea_collisions": 0,
236 "idea_pred_target_corr": 0.886642980654682
237 },
238 {
239 "seed": 2,
240 "baseline_metric": 0.09898804873228073,
241 "idea_metric": 0.0886211097240448,
242 "baseline_collisions": 19,
243 "idea_collisions": 0,
244 "idea_pred_target_corr": 0.8899489630055591
245 },
246 {
247 "seed": 3,
248 "baseline_metric": 0.09606796503067017,
249 "idea_metric": 0.08095572143793106,
250 "baseline_collisions": 19,
251 "idea_collisions": 0,
252 "idea_pred_target_corr": 0.8968686834235331
253 },
254 {
255 "seed": 4,
256 "baseline_metric": 0.10214304178953171,
257 "idea_metric": 0.09157451242208481,
258 "baseline_collisions": 19,
259 "idea_collisions": 0,
260 "idea_pred_target_corr": 0.8833831997989209
261 },
262 {
263 "seed": 5,
264 "baseline_metric": 0.09961570054292679,
265 "idea_metric": 0.07902113348245621,
266 "baseline_collisions": 19,
267 "idea_collisions": 0,
268 "idea_pred_target_corr": 0.8995094696825414
269 },
270 {
271 "seed": 6,
272 "baseline_metric": 0.09206581115722656,
273 "idea_metric": 0.0880790650844574,
274 "baseline_collisions": 19,
275 "idea_collisions": 0,
276 "idea_pred_target_corr": 0.8922238907368641
277 },
278 {
279 "seed": 7,
280 "baseline_metric": 0.09584151953458786,
281 "idea_metric": 0.08560939878225327,
282 "baseline_collisions": 19,
283 "idea_collisions": 0,
284 "idea_pred_target_corr": 0.8906957622074757
285 }
286 ],
287 "confirmed": true
288 }
289}