Spectral quadrature features / bench_report.json

Failed on benchmark

Raw ⬇ ZIP
 1{
 2  "bench_version": 1,
 3  "track": "tabular",
 4  "model": "fourier_mlp_shared",
 5  "metric_direction": "lower is better",
 6  "n_seeds": 8,
 7  "baseline": {
 8    "best_cfg": {
 9      "lr": 0.003
10    },
11    "sweep": [
12      {
13        "cfg": {
14          "lr": 0.001
15        },
16        "mean": 33.56439447402954
17      },
18      {
19        "cfg": {
20          "lr": 0.003
21        },
22        "mean": 22.75266695022583
23      },
24      {
25        "cfg": {
26          "lr": 0.01
27        },
28        "mean": 22.805188179016113
29      }
30    ],
31    "full": {
32      "mean": 23.26982045173645,
33      "std": 2.8220737062934167,
34      "per_seed": [
35        23.454627990722656,
36        20.045284271240234,
37        23.504892349243164,
38        24.005863189697266,
39        23.5848388671875,
40        27.756813049316406,
41        18.11393165588379,
42        25.692312240600586
43      ],
44      "n": 8
45    }
46  },
47  "idea": {
48    "mean": 24.201251745224,
49    "std": 1.9567082483775788,
50    "per_seed": [
51      25.041019439697266,
52      21.52345848083496,
53      25.802366256713867,
54      22.822996139526367,
55      21.128692626953125,
56      26.736886978149414,
57      25.048274993896484,
58      25.506319046020508
59    ],
60    "n": 8
61  },
62  "comparison": {
63    "delta_mean": 0.9314312934875488,
64    "idea_wins": 4,
65    "n_pairs": 8,
66    "per_seed_diffs": [
67      1.5863914489746094,
68      1.4781742095947266,
69      2.297473907470703,
70      -1.1828670501708984,
71      -2.456146240234375,
72      -1.0199260711669922,
73      6.934343338012695,
74      -0.18599319458007812
75    ],
76    "p_value": 0.43755,
77    "mde": 2.4404744399584257,
78    "mde_rel_pct": 10.487723551714435,
79    "verdict": "no significant win",
80    "system_worked": false
81  },
82  "notes": "Tabular is the matched built-in track for an input embedding/kernel feature intervention.",
83  "mechanism_signature": {
84    "prediction": "deterministic quadrature has lower top-10 RBF Gram eigenvalue error",
85    "predicted_baseline_top10_relative_error": 1.4234343767166138,
86    "predicted_idea_top10_relative_error": 4.320810317993164,
87    "observed_delta_idea_minus_baseline": 2.8973759412765503,
88    "confirmed": false
89  }
90}