Context-free denoiser with analytic quadratic score injection / bench_report.json
Beats tuned baseline
1{
2 "bench_version": 1,
3 "track": "quadratic_context_multitoken",
4 "model": "local_mlp",
5 "metric_direction": "lower is better",
6 "n_seeds": 8,
7 "baseline": {
8 "best_cfg": {
9 "lr": 0.003
10 },
11 "sweep": [
12 {
13 "cfg": {
14 "lr": 0.001
15 },
16 "mean": 0.053421652875840664
17 },
18 {
19 "cfg": {
20 "lr": 0.003
21 },
22 "mean": 0.05309625621885061
23 },
24 {
25 "cfg": {
26 "lr": 0.009
27 },
28 "mean": 0.053741672076284885
29 }
30 ],
31 "full": {
32 "mean": 0.054212482180446386,
33 "std": 0.007988832816369695,
34 "per_seed": [
35 0.06300174444913864,
36 0.05289159715175629,
37 0.04336535930633545,
38 0.05312632396817207,
39 0.06166467070579529,
40 0.0440090112388134,
41 0.0498502142727375,
42 0.06579093635082245
43 ],
44 "n": 8
45 },
46 "all_full": [
47 {
48 "per_seed": [
49 0.055687107145786285,
50 0.05908738821744919,
51 0.047396738082170486,
52 0.051515378057956696,
53 0.06218918785452843,
54 0.057241544127464294,
55 0.05223905295133591,
56 0.0556969977915287
57 ],
58 "mean": 0.0551316742785275,
59 "std": 0.004656944311225746,
60 "lr": 0.001,
61 "epochs": 18,
62 "ratios": [
63 0.6952053904533386,
64 0.7031234502792358,
65 0.6710729598999023,
66 0.6854270696640015,
67 0.7208964824676514,
68 0.7045612335205078,
69 0.6924821734428406,
70 0.6988305449485779
71 ]
72 },
73 {
74 "per_seed": [
75 0.06300174444913864,
76 0.05289159715175629,
77 0.04336535930633545,
78 0.05312632396817207,
79 0.06166467070579529,
80 0.0440090112388134,
81 0.0498502142727375,
82 0.06579093635082245
83 ],
84 "mean": 0.054212482180446386,
85 "std": 0.008540421519734782,
86 "lr": 0.003,
87 "epochs": 18,
88 "ratios": [
89 0.7323943376541138,
90 0.6982712149620056,
91 0.6692168116569519,
92 0.702392041683197,
93 0.7244734764099121,
94 0.6718965172767639,
95 0.6918891668319702,
96 0.7385606169700623
97 ]
98 },
99 {
100 "per_seed": [
101 0.0538078248500824,
102 0.050708189606666565,
103 0.05362054705619812,
104 0.05683012679219246,
105 0.05819164216518402,
106 0.050066135823726654,
107 0.04446486756205559,
108 0.059522490948438644
109 ],
110 "mean": 0.053401478100568056,
111 "std": 0.0049370686751292045,
112 "lr": 0.009,
113 "epochs": 18,
114 "ratios": [
115 0.7059165835380554,
116 0.6909855008125305,
117 0.6965495944023132,
118 0.7143308520317078,
119 0.7151844501495361,
120 0.6841487884521484,
121 0.6683564186096191,
122 0.7204694151878357
123 ]
124 }
125 ]
126 },
127 "idea": {
128 "per_seed": [
129 0.0006221848307177424,
130 0.0005909778992645442,
131 0.00076779950177297,
132 0.0006266951677389443,
133 0.000846916635055095,
134 0.0007931980071589351,
135 0.0008056167862378061,
136 0.0007610649918206036
137 ],
138 "mean": 0.0007268067274708301,
139 "std": 9.806469420502945e-05,
140 "lr": 0.009,
141 "epochs": 18,
142 "ratios": [
143 0.9858091473579407,
144 0.9577128887176514,
145 0.9639421105384827,
146 0.9625136852264404,
147 0.9858413338661194,
148 0.969452440738678,
149 0.95078045129776,
150 0.9798619747161865
151 ],
152 "sweep": [
153 {
154 "per_seed": [
155 0.005637110210955143,
156 0.004224653821438551,
157 0.0062506780959665775,
158 0.005489944480359554,
159 0.0034460758324712515,
160 0.004024875815957785,
161 0.003128137206658721,
162 0.0033030833583325148
163 ],
164 "mean": 0.004438069852767512,
165 "std": 0.0011972666719732973,
166 "lr": 0.001,
167 "epochs": 18,
168 "ratios": [
169 0.8957160115242004,
170 0.9036473035812378,
171 0.9012296199798584,
172 0.8910174369812012,
173 0.8902207612991333,
174 0.8937434554100037,
175 0.9013668894767761,
176 0.9114634394645691
177 ]
178 },
179 {
180 "per_seed": [
181 0.000794272986240685,
182 0.0007921864162199199,
183 0.001272958004847169,
184 0.000900555111002177,
185 0.0006123684579506516,
186 0.0008760863565839827,
187 0.0007426603115163743,
188 0.0007867970853112638
189 ],
190 "mean": 0.0008472355912090279,
191 "std": 0.00019291068730023744,
192 "lr": 0.003,
193 "epochs": 18,
194 "ratios": [
195 0.9768263101577759,
196 0.9619719982147217,
197 0.9295895099639893,
198 0.9479344487190247,
199 0.9476742744445801,
200 0.9384970664978027,
201 0.9460005164146423,
202 0.9515169262886047
203 ]
204 },
205 {
206 "per_seed": [
207 0.0006221848307177424,
208 0.0005909778992645442,
209 0.00076779950177297,
210 0.0006266951677389443,
211 0.000846916635055095,
212 0.0007931980071589351,
213 0.0008056167862378061,
214 0.0007610649918206036
215 ],
216 "mean": 0.0007268067274708301,
217 "std": 9.806469420502945e-05,
218 "lr": 0.009,
219 "epochs": 18,
220 "ratios": [
221 0.9858091473579407,
222 0.9577128887176514,
223 0.9639421105384827,
224 0.9625136852264404,
225 0.9858413338661194,
226 0.969452440738678,
227 0.95078045129776,
228 0.9798619747161865
229 ]
230 }
231 ]
232 },
233 "comparison": {
234 "delta_mean": -0.053485675452975556,
235 "idea_wins": 8,
236 "n_pairs": 8,
237 "per_seed_diffs": [
238 -0.0623795596184209,
239 -0.05230061925249174,
240 -0.04259755980456248,
241 -0.05249962880043313,
242 -0.06081775407074019,
243 -0.043215813231654465,
244 -0.0490445974864997,
245 -0.06502987135900185
246 ],
247 "p_value": 0.0081,
248 "mde": 0.007152833233489385,
249 "mde_rel_pct": 13.194070711761844,
250 "verdict": "idea better (significant)",
251 "system_worked": true
252 },
253 "mechanism_signature": {
254 "predicted_shrink": 0.4,
255 "observed_nn_output_ratio": 0.9694892540574074,
256 "relative_error": 1.4237231351435182,
257 "confirmed": false
258 },
259 "custom_track": {
260 "name": "quadratic_context_multitoken",
261 "file": "quadratic_context_track.py",
262 "domain": "diffusion-sampling"
263 },
264 "seconds": 17.945859909057617
265}