Hermite-critical residual initialization / bench_report.json
Failed on benchmark
1{
2 "bench_version": 1,
3 "track": "tabular",
4 "model": "local_residual_mlp",
5 "metric_direction": "lower is better",
6 "n_seeds": 8,
7 "baseline": {
8 "sweep": [
9 {
10 "cfg": {
11 "lr": 0.001,
12 "epochs": 18,
13 "exponent": 0.5
14 },
15 "mean": 7.946774959564209
16 },
17 {
18 "cfg": {
19 "lr": 0.001,
20 "epochs": 18,
21 "exponent": 0.8
22 },
23 "mean": 10.225590229034424
24 },
25 {
26 "cfg": {
27 "lr": 0.003,
28 "epochs": 18,
29 "exponent": 0.5
30 },
31 "mean": 7.576857328414917
32 },
33 {
34 "cfg": {
35 "lr": 0.003,
36 "epochs": 18,
37 "exponent": 0.8
38 },
39 "mean": 7.316884994506836
40 },
41 {
42 "cfg": {
43 "lr": 0.006,
44 "epochs": 18,
45 "exponent": 0.5
46 },
47 "mean": 6.160425186157227
48 },
49 {
50 "cfg": {
51 "lr": 0.006,
52 "epochs": 18,
53 "exponent": 0.8
54 },
55 "mean": 7.347582578659058
56 }
57 ],
58 "best_cfg": {
59 "lr": 0.006,
60 "epochs": 18,
61 "exponent": 0.5
62 },
63 "full": {
64 "cfg": {
65 "lr": 0.006,
66 "epochs": 18,
67 "exponent": 0.5
68 },
69 "per_seed": [
70 5.471584320068359,
71 6.244831562042236,
72 7.435494422912598,
73 5.489790439605713,
74 7.0974907875061035,
75 5.940451622009277,
76 7.106790065765381,
77 7.513341903686523
78 ],
79 "mean": 6.537471890449524,
80 "std": 0.7962674958223888,
81 "n": 8
82 }
83 },
84 "idea": {
85 "cfg": {
86 "lr": 0.006,
87 "epochs": 18,
88 "exponent": 0.7
89 },
90 "mean": 7.036507606506348,
91 "per_seed": [
92 7.978086948394775,
93 6.377083778381348,
94 7.57796049118042,
95 5.985846996307373,
96 7.230166435241699,
97 6.412862300872803,
98 7.17480993270874,
99 7.555243968963623
100 ],
101 "std": 0.6550880229661826,
102 "n": 8
103 },
104 "comparison": {
105 "delta_mean": 0.49903571605682373,
106 "idea_wins": 0,
107 "n_pairs": 8,
108 "per_seed_diffs": [
109 2.506502628326416,
110 0.13225221633911133,
111 0.14246606826782227,
112 0.49605655670166016,
113 0.1326756477355957,
114 0.4724106788635254,
115 0.06801986694335938,
116 0.04190206527709961
117 ],
118 "p_value": 0.0081,
119 "mde": 0.6939341374663044,
120 "mde_rel_pct": 10.61471695931971,
121 "verdict": "idea worse (significant)",
122 "system_worked": false
123 },
124 "math_check": {
125 "theory_H": 0.8,
126 "observed_H": 0.8212929503641088,
127 "ns": [
128 16,
129 32,
130 64,
131 128,
132 256
133 ],
134 "scaled_sum_rms": [
135 1.3914831618943797,
136 1.2675475655440305,
137 1.4158369110940938,
138 1.2996234839260772,
139 1.4794503561406334
140 ],
141 "rms_ratio_last_first": 1.0632182958840077,
142 "passed": true
143 },
144 "protocol_note": "Matched tabular initialization track: same residual MLP, data, Adam, epochs, batch, and lr union.",
145 "mechanism_signature": {
146 "prediction": "long-memory gates retain positive adjacent correlation; critical lambda keeps scaled sums O(1)",
147 "predicted_adjacent_corr": 0.757858283255199,
148 "observed_adjacent_corr_mean": 0.03618130783796214,
149 "observations": [
150 {
151 "seed": 0,
152 "baseline_adjacent_gate_corr": -0.0724730360728216,
153 "idea_adjacent_gate_corr": 0.017231813071073975,
154 "baseline_test_pred_std": 4.221856117248535,
155 "idea_test_pred_std": 3.7397565841674805,
156 "baseline_final_train_loss": 6.201660385131836,
157 "idea_final_train_loss": 6.991382598876953
158 },
159 {
160 "seed": 1,
161 "baseline_adjacent_gate_corr": 0.01320452253024341,
162 "idea_adjacent_gate_corr": 0.45370841081914715,
163 "baseline_test_pred_std": 4.062014579772949,
164 "idea_test_pred_std": 4.043881416320801,
165 "baseline_final_train_loss": 4.762135734558106,
166 "idea_final_train_loss": 5.053467597961426
167 },
168 {
169 "seed": 2,
170 "baseline_adjacent_gate_corr": 0.20420422523092288,
171 "idea_adjacent_gate_corr": 0.6252111773933212,
172 "baseline_test_pred_std": 4.786465644836426,
173 "idea_test_pred_std": 4.8650898933410645,
174 "baseline_final_train_loss": 3.884450912475586,
175 "idea_final_train_loss": 4.2415578651428225
176 },
177 {
178 "seed": 3,
179 "baseline_adjacent_gate_corr": -0.7082938951127355,
180 "idea_adjacent_gate_corr": -0.6668175384417161,
181 "baseline_test_pred_std": 4.167241096496582,
182 "idea_test_pred_std": 4.061396598815918,
183 "baseline_final_train_loss": 6.076703529357911,
184 "idea_final_train_loss": 6.359731369018554
185 },
186 {
187 "seed": 4,
188 "baseline_adjacent_gate_corr": -0.7208059938362777,
189 "idea_adjacent_gate_corr": -0.6612016062075473,
190 "baseline_test_pred_std": 3.6289286613464355,
191 "idea_test_pred_std": 3.551570415496826,
192 "baseline_final_train_loss": 5.892031574249268,
193 "idea_final_train_loss": 6.300574836730957
194 },
195 {
196 "seed": 5,
197 "baseline_adjacent_gate_corr": 0.26950874374172223,
198 "idea_adjacent_gate_corr": 0.2965993789633692,
199 "baseline_test_pred_std": 4.365800380706787,
200 "idea_test_pred_std": 4.453507900238037,
201 "baseline_final_train_loss": 4.534175910949707,
202 "idea_final_train_loss": 4.923007583618164
203 },
204 {
205 "seed": 6,
206 "baseline_adjacent_gate_corr": -0.3201237467772104,
207 "idea_adjacent_gate_corr": -0.32051733449908076,
208 "baseline_test_pred_std": 3.79843807220459,
209 "idea_test_pred_std": 3.9083645343780518,
210 "baseline_final_train_loss": 7.1678547668457036,
211 "idea_final_train_loss": 7.365681228637695
212 },
213 {
214 "seed": 7,
215 "baseline_adjacent_gate_corr": 0.4083689217021288,
216 "idea_adjacent_gate_corr": 0.5452361616051298,
217 "baseline_test_pred_std": 4.7109575271606445,
218 "idea_test_pred_std": 4.555789947509766,
219 "baseline_final_train_loss": 3.857210865020752,
220 "idea_final_train_loss": 4.864171485900879
221 }
222 ],
223 "confirmed": false
224 }
225}