Resolution-aware operator data budget / bench_report.json
Running benchmark…
1{
2 "bench_version": 1,
3 "track": "resolution_hilbert_regression",
4 "model": "mlp_tiny",
5 "metric_direction": "lower is better",
6 "n_seeds": 8,
7 "baseline": {
8 "sweep": [
9 {
10 "cfg": {
11 "lr": 0.001,
12 "n_pairs": 4,
13 "m": 64,
14 "epochs": 12
15 },
16 "mean": 0.4742010235786438
17 },
18 {
19 "cfg": {
20 "lr": 0.001,
21 "n_pairs": 6,
22 "m": 64,
23 "epochs": 12
24 },
25 "mean": 0.5084983855485916
26 },
27 {
28 "cfg": {
29 "lr": 0.001,
30 "n_pairs": 8,
31 "m": 64,
32 "epochs": 12
33 },
34 "mean": 0.49686431139707565
35 },
36 {
37 "cfg": {
38 "lr": 0.001,
39 "n_pairs": 16,
40 "m": 64,
41 "epochs": 12
42 },
43 "mean": 0.4806900769472122
44 },
45 {
46 "cfg": {
47 "lr": 0.001,
48 "n_pairs": 32,
49 "m": 64,
50 "epochs": 12
51 },
52 "mean": 0.45177099108695984
53 },
54 {
55 "cfg": {
56 "lr": 0.001,
57 "n_pairs": 64,
58 "m": 64,
59 "epochs": 12
60 },
61 "mean": 0.3843056559562683
62 },
63 {
64 "cfg": {
65 "lr": 0.001,
66 "n_pairs": 182,
67 "m": 64,
68 "epochs": 12
69 },
70 "mean": 0.28786445409059525
71 },
72 {
73 "cfg": {
74 "lr": 0.001,
75 "n_pairs": 400,
76 "m": 64,
77 "epochs": 12
78 },
79 "mean": 0.11739674024283886
80 },
81 {
82 "cfg": {
83 "lr": 0.003,
84 "n_pairs": 4,
85 "m": 64,
86 "epochs": 12
87 },
88 "mean": 0.4902075082063675
89 },
90 {
91 "cfg": {
92 "lr": 0.003,
93 "n_pairs": 6,
94 "m": 64,
95 "epochs": 12
96 },
97 "mean": 0.5222374796867371
98 },
99 {
100 "cfg": {
101 "lr": 0.003,
102 "n_pairs": 8,
103 "m": 64,
104 "epochs": 12
105 },
106 "mean": 0.49188823997974396
107 },
108 {
109 "cfg": {
110 "lr": 0.003,
111 "n_pairs": 16,
112 "m": 64,
113 "epochs": 12
114 },
115 "mean": 0.43465904891490936
116 },
117 {
118 "cfg": {
119 "lr": 0.003,
120 "n_pairs": 32,
121 "m": 64,
122 "epochs": 12
123 },
124 "mean": 0.3434709310531616
125 },
126 {
127 "cfg": {
128 "lr": 0.003,
129 "n_pairs": 64,
130 "m": 64,
131 "epochs": 12
132 },
133 "mean": 0.2486615851521492
134 },
135 {
136 "cfg": {
137 "lr": 0.003,
138 "n_pairs": 182,
139 "m": 64,
140 "epochs": 12
141 },
142 "mean": 0.1157324742525816
143 },
144 {
145 "cfg": {
146 "lr": 0.003,
147 "n_pairs": 400,
148 "m": 64,
149 "epochs": 12
150 },
151 "mean": 0.05876756366342306
152 },
153 {
154 "cfg": {
155 "lr": 0.01,
156 "n_pairs": 4,
157 "m": 64,
158 "epochs": 12
159 },
160 "mean": 0.5567187815904617
161 },
162 {
163 "cfg": {
164 "lr": 0.01,
165 "n_pairs": 6,
166 "m": 64,
167 "epochs": 12
168 },
169 "mean": 0.5994730815291405
170 },
171 {
172 "cfg": {
173 "lr": 0.01,
174 "n_pairs": 8,
175 "m": 64,
176 "epochs": 12
177 },
178 "mean": 0.5139493346214294
179 },
180 {
181 "cfg": {
182 "lr": 0.01,
183 "n_pairs": 16,
184 "m": 64,
185 "epochs": 12
186 },
187 "mean": 0.43380651623010635
188 },
189 {
190 "cfg": {
191 "lr": 0.01,
192 "n_pairs": 32,
193 "m": 64,
194 "epochs": 12
195 },
196 "mean": 0.32234662771224976
197 },
198 {
199 "cfg": {
200 "lr": 0.01,
201 "n_pairs": 64,
202 "m": 64,
203 "epochs": 12
204 },
205 "mean": 0.20379487425088882
206 },
207 {
208 "cfg": {
209 "lr": 0.01,
210 "n_pairs": 182,
211 "m": 64,
212 "epochs": 12
213 },
214 "mean": 0.08751582726836205
215 },
216 {
217 "cfg": {
218 "lr": 0.01,
219 "n_pairs": 400,
220 "m": 64,
221 "epochs": 12
222 },
223 "mean": 0.04640552960336208
224 }
225 ],
226 "best_cfg": {
227 "lr": 0.01,
228 "n_pairs": 400,
229 "m": 64,
230 "epochs": 12
231 },
232 "full": {
233 "per_seed": [
234 0.045129235833883286,
235 0.04374578222632408,
236 0.05521933361887932,
237 0.04152776673436165,
238 0.04662596434354782,
239 0.03831035643815994,
240 0.05773128941655159,
241 0.04768511280417442
242 ],
243 "mean": 0.046996855176985264,
244 "std": 0.006152968336672968,
245 "n": 8
246 }
247 },
248 "idea": {
249 "config": {
250 "lr": 0.01,
251 "kappa": 1.5,
252 "n_pairs": 400,
253 "m": 64,
254 "epochs": 12
255 },
256 "per_seed": [
257 0.045129235833883286,
258 0.04374578222632408,
259 0.05521933361887932,
260 0.04152776673436165,
261 0.04662596434354782,
262 0.03831035643815994,
263 0.05773128941655159,
264 0.04768511280417442
265 ],
266 "mean": 0.046996855176985264,
267 "std": 0.006152968336672968,
268 "n": 8,
269 "details": [
270 {
271 "seed": 0,
272 "baseline": {
273 "metric": 0.045129235833883286,
274 "seconds": 0.13750473599998259,
275 "n_pairs": 400,
276 "m": 64,
277 "lr": 0.01,
278 "train_last": 0.01252928800880909
279 },
280 "idea": {
281 "metric": 0.045129235833883286,
282 "seconds": 0.138005157000066,
283 "n_pairs": 400,
284 "m": 64,
285 "lr": 0.01,
286 "train_last": 0.01252928800880909
287 }
288 },
289 {
290 "seed": 1,
291 "baseline": {
292 "metric": 0.04374578222632408,
293 "seconds": 0.13704957300001297,
294 "n_pairs": 400,
295 "m": 64,
296 "lr": 0.01,
297 "train_last": 0.01171974055469036
298 },
299 "idea": {
300 "metric": 0.04374578222632408,
301 "seconds": 0.1325516530000641,
302 "n_pairs": 400,
303 "m": 64,
304 "lr": 0.01,
305 "train_last": 0.01171974055469036
306 }
307 },
308 {
309 "seed": 2,
310 "baseline": {
311 "metric": 0.05521933361887932,
312 "seconds": 0.13162763799994082,
313 "n_pairs": 400,
314 "m": 64,
315 "lr": 0.01,
316 "train_last": 0.006767850443720818
317 },
318 "idea": {
319 "metric": 0.05521933361887932,
320 "seconds": 0.14726283699997111,
321 "n_pairs": 400,
322 "m": 64,
323 "lr": 0.01,
324 "train_last": 0.006767850443720818
325 }
326 },
327 {
328 "seed": 3,
329 "baseline": {
330 "metric": 0.04152776673436165,
331 "seconds": 0.13383954699997958,
332 "n_pairs": 400,
333 "m": 64,
334 "lr": 0.01,
335 "train_last": 0.007624506056308746
336 },
337 "idea": {
338 "metric": 0.04152776673436165,
339 "seconds": 0.13501674099995853,
340 "n_pairs": 400,
341 "m": 64,
342 "lr": 0.01,
343 "train_last": 0.007624506056308746
344 }
345 },
346 {
347 "seed": 4,
348 "baseline": {
349 "metric": 0.04662596434354782,
350 "seconds": 0.1409590709999975,
351 "n_pairs": 400,
352 "m": 64,
353 "lr": 0.01,
354 "train_last": 0.007026923410594463
355 },
356 "idea": {
357 "metric": 0.04662596434354782,
358 "seconds": 0.13288459200009584,
359 "n_pairs": 400,
360 "m": 64,
361 "lr": 0.01,
362 "train_last": 0.007026923410594463
363 }
364 },
365 {
366 "seed": 5,
367 "baseline": {
368 "metric": 0.03831035643815994,
369 "seconds": 0.13310180199994193,
370 "n_pairs": 400,
371 "m": 64,
372 "lr": 0.01,
373 "train_last": 0.007159473076462745
374 },
375 "idea": {
376 "metric": 0.03831035643815994,
377 "seconds": 0.132771236999929,
378 "n_pairs": 400,
379 "m": 64,
380 "lr": 0.01,
381 "train_last": 0.007159473076462745
382 }
383 },
384 {
385 "seed": 6,
386 "baseline": {
387 "metric": 0.05773128941655159,
388 "seconds": 0.1347488249999742,
389 "n_pairs": 400,
390 "m": 64,
391 "lr": 0.01,
392 "train_last": 0.007970670312643051
393 },
394 "idea": {
395 "metric": 0.05773128941655159,
396 "seconds": 0.13416682800004764,
397 "n_pairs": 400,
398 "m": 64,
399 "lr": 0.01,
400 "train_last": 0.007970670312643051
401 }
402 },
403 {
404 "seed": 7,
405 "baseline": {
406 "metric": 0.04768511280417442,
407 "seconds": 0.1469870290000017,
408 "n_pairs": 400,
409 "m": 64,
410 "lr": 0.01,
411 "train_last": 0.0068675529211759566
412 },
413 "idea": {
414 "metric": 0.04768511280417442,
415 "seconds": 0.1359896449999951,
416 "n_pairs": 400,
417 "m": 64,
418 "lr": 0.01,
419 "train_last": 0.0068675529211759566
420 }
421 }
422 ]
423 },
424 "comparison": {
425 "delta_mean": 0.0,
426 "idea_wins": 0,
427 "n_pairs": 8,
428 "per_seed_diffs": [
429 0.0,
430 0.0,
431 0.0,
432 0.0,
433 0.0,
434 0.0,
435 0.0,
436 0.0
437 ],
438 "p_value": 1.0,
439 "mde": 0.0,
440 "mde_rel_pct": 0.0,
441 "verdict": "no measurable effect",
442 "system_worked": false
443 },
444 "math_check": {
445 "m": [
446 4,
447 8,
448 16,
449 32,
450 64
451 ],
452 "fill_distance": [
453 0.16666666666666663,
454 0.0714285714285714,
455 0.033333333333333326,
456 0.016129032258064502,
457 0.007936507936507908
458 ],
459 "theory": [
460 0.16666666666666666,
461 0.07142857142857142,
462 0.03333333333333333,
463 0.016129032258064516,
464 0.007936507936507936
465 ],
466 "loglog_slope": -1.0931476233886805,
467 "max_theory_error": 2.7755575615628914e-17,
468 "confirmed": true
469 },
470 "idea_sweep": [
471 {
472 "cfg": {
473 "lr": 0.001,
474 "kappa": 0.5,
475 "n_pairs": 8,
476 "m": 64,
477 "epochs": 12
478 },
479 "mean": 0.49686431139707565
480 },
481 {
482 "cfg": {
483 "lr": 0.001,
484 "kappa": 1.0,
485 "n_pairs": 64,
486 "m": 64,
487 "epochs": 12
488 },
489 "mean": 0.3843056559562683
490 },
491 {
492 "cfg": {
493 "lr": 0.001,
494 "kappa": 1.5,
495 "n_pairs": 400,
496 "m": 64,
497 "epochs": 12
498 },
499 "mean": 0.11739674024283886
500 },
501 {
502 "cfg": {
503 "lr": 0.003,
504 "kappa": 0.5,
505 "n_pairs": 8,
506 "m": 64,
507 "epochs": 12
508 },
509 "mean": 0.49188823997974396
510 },
511 {
512 "cfg": {
513 "lr": 0.003,
514 "kappa": 1.0,
515 "n_pairs": 64,
516 "m": 64,
517 "epochs": 12
518 },
519 "mean": 0.2486615851521492
520 },
521 {
522 "cfg": {
523 "lr": 0.003,
524 "kappa": 1.5,
525 "n_pairs": 400,
526 "m": 64,
527 "epochs": 12
528 },
529 "mean": 0.05876756366342306
530 },
531 {
532 "cfg": {
533 "lr": 0.01,
534 "kappa": 0.5,
535 "n_pairs": 8,
536 "m": 64,
537 "epochs": 12
538 },
539 "mean": 0.5139493346214294
540 },
541 {
542 "cfg": {
543 "lr": 0.01,
544 "kappa": 1.0,
545 "n_pairs": 64,
546 "m": 64,
547 "epochs": 12
548 },
549 "mean": 0.20379487425088882
550 },
551 {
552 "cfg": {
553 "lr": 0.01,
554 "kappa": 1.5,
555 "n_pairs": 400,
556 "m": 64,
557 "epochs": 12
558 },
559 "mean": 0.04640552960336208
560 }
561 ],
562 "mechanism_signature": {
563 "trained_model_observations": [
564 {
565 "m": 16,
566 "n_pairs": 16,
567 "test_mse": 0.4301315248012543,
568 "pred_std": 0.3446804881095886,
569 "truth_std": 0.6841227412223816,
570 "pred_truth_corr": 0.36960441688575535
571 },
572 {
573 "m": 32,
574 "n_pairs": 32,
575 "test_mse": 0.30473387241363525,
576 "pred_std": 0.4022413194179535,
577 "truth_std": 0.6841227412223816,
578 "pred_truth_corr": 0.6004599391012871
579 },
580 {
581 "m": 64,
582 "n_pairs": 64,
583 "test_mse": 0.21641530096530914,
584 "pred_std": 0.39895883202552795,
585 "truth_std": 0.6841227412223816,
586 "pred_truth_corr": 0.7569471296122684
587 }
588 ],
589 "prediction_error_at_larger_resolution_lower": true,
590 "confirmed": true
591 }
592}