Walk-on-Spheres stochastic target layer / bench_report.json
Mechanism confirmed, baseline not beaten
1{
2 "bench_version": 1,
3 "track": "poisson_boundary",
4 "model": "mlp_tiny",
5 "metric_direction": "lower is better",
6 "n_seeds": 8,
7 "baseline": {
8 "best_cfg": {
9 "lr": 0.01
10 },
11 "sweep": [
12 {
13 "cfg": {
14 "lr": 0.001
15 },
16 "mean": 0.046625981107354164
17 },
18 {
19 "cfg": {
20 "lr": 0.003
21 },
22 "mean": 0.005821306491270661
23 },
24 {
25 "cfg": {
26 "lr": 0.01
27 },
28 "mean": 0.0026237370329909027
29 }
30 ],
31 "full": {
32 "mean": 0.0022902884520590305,
33 "std": 0.0008475947478694112,
34 "per_seed": [
35 0.002140381373465061,
36 0.0017085936851799488,
37 0.0028188657015562057,
38 0.003827107371762395,
39 0.0007765397895127535,
40 0.0018183346837759018,
41 0.002519753761589527,
42 0.002712731249630451
43 ],
44 "n": 8
45 }
46 },
47 "idea": {
48 "mean": 0.01020078727742657,
49 "std": 0.00296426428649209,
50 "per_seed": [
51 0.009175088256597519,
52 0.007033985573798418,
53 0.006878904066979885,
54 0.015077905729413033,
55 0.014966735616326332,
56 0.009611713699996471,
57 0.009160290472209454,
58 0.009701674804091454
59 ],
60 "n": 8
61 },
62 "comparison": {
63 "delta_mean": 0.00791049882536754,
64 "idea_wins": 0,
65 "n_pairs": 8,
66 "per_seed_diffs": [
67 0.007034706883132458,
68 0.005325391888618469,
69 0.004060038365423679,
70 0.011250798357650638,
71 0.014190195826813579,
72 0.00779337901622057,
73 0.0066405367106199265,
74 0.006988943554461002
75 ],
76 "p_value": 0.0081,
77 "mde": 0.0027432218817820646,
78 "mde_rel_pct": 119.77626134017463,
79 "verdict": "idea worse (significant)",
80 "system_worked": false
81 },
82 "mechanism_signature": {
83 "idea_config": {
84 "lr": 0.003
85 },
86 "idea_sweep": [
87 {
88 "cfg": {
89 "lr": 0.001
90 },
91 "result": {
92 "mean": 0.052139872685074806,
93 "std": 0.010604419760995165,
94 "per_seed": [
95 0.055915020406246185,
96 0.04067733511328697,
97 0.048677653074264526,
98 0.06918065994977951,
99 0.03635910525918007,
100 0.0652485340833664,
101 0.05410538986325264,
102 0.04695528373122215
103 ],
104 "n": 8
105 }
106 },
107 {
108 "cfg": {
109 "lr": 0.003
110 },
111 "result": {
112 "mean": 0.01020078727742657,
113 "std": 0.00296426428649209,
114 "per_seed": [
115 0.009175088256597519,
116 0.007033985573798418,
117 0.006878904066979885,
118 0.015077905729413033,
119 0.014966735616326332,
120 0.009611713699996471,
121 0.009160290472209454,
122 0.009701674804091454
123 ],
124 "n": 8
125 }
126 },
127 {
128 "cfg": {
129 "lr": 0.01
130 },
131 "result": {
132 "mean": 0.011461915099062026,
133 "std": 0.0024442795764689383,
134 "per_seed": [
135 0.009399672970175743,
136 0.0106451865285635,
137 0.009206349961459637,
138 0.012322325259447098,
139 0.01035053189843893,
140 0.011077546514570713,
141 0.011263057589530945,
142 0.01743065007030964
143 ],
144 "n": 8
145 }
146 }
147 ],
148 "math_check": {
149 "sphere_second_moment_increment": 0.1600427977741897,
150 "predicted_increment": 0.16000000000000003,
151 "relative_increment_error": 0.00026748608868542434,
152 "variance_scaling": [
153 {
154 "K": 1,
155 "variance": 0.4991379976272583,
156 "K_times_variance": 0.4991379976272583
157 },
158 {
159 "K": 2,
160 "variance": 0.24831385910511017,
161 "K_times_variance": 0.49662771821022034
162 },
163 {
164 "K": 4,
165 "variance": 0.12296506762504578,
166 "K_times_variance": 0.4918602705001831
167 },
168 {
169 "K": 8,
170 "variance": 0.060902342200279236,
171 "K_times_variance": 0.4872187376022339
172 },
173 {
174 "K": 16,
175 "variance": 0.03029288910329342,
176 "K_times_variance": 0.4846862256526947
177 },
178 {
179 "K": 32,
180 "variance": 0.014903044328093529,
181 "K_times_variance": 0.4768974184989929
182 },
183 {
184 "K": 64,
185 "variance": 0.007048895116895437,
186 "K_times_variance": 0.451129287481308
187 },
188 {
189 "K": 128,
190 "variance": 0.0035142002161592245,
191 "K_times_variance": 0.44981762766838074
192 }
193 ],
194 "K_product_cv": 0.03766105478867267,
195 "confirmed": true
196 },
197 "mechanism_signature": {
198 "prediction": "WOS stochastic supervision does not create a systematic output mean shift",
199 "predicted_shift": 0.0,
200 "observed_shift_mean": -0.013755707695963793,
201 "observed_shift_abs_mean": 0.028182248890516348,
202 "trained_systems": [
203 {
204 "seed": 0,
205 "baseline_mse": 0.005913021042943001,
206 "idea_mse": 0.009175088256597519,
207 "trained_prediction_mean_shift_idea_minus_baseline": 0.00194565171841532
208 },
209 {
210 "seed": 1,
211 "baseline_mse": 0.003503378015011549,
212 "idea_mse": 0.007033985573798418,
213 "trained_prediction_mean_shift_idea_minus_baseline": -0.02376757189631462
214 },
215 {
216 "seed": 2,
217 "baseline_mse": 0.003274704795330763,
218 "idea_mse": 0.006878904066979885,
219 "trained_prediction_mean_shift_idea_minus_baseline": 0.0016674669459462166
220 },
221 {
222 "seed": 3,
223 "baseline_mse": 0.010594122111797333,
224 "idea_mse": 0.015077905729413033,
225 "trained_prediction_mean_shift_idea_minus_baseline": 0.023475755006074905
226 },
227 {
228 "seed": 4,
229 "baseline_mse": 0.0036502110306173563,
230 "idea_mse": 0.014966735616326332,
231 "trained_prediction_mean_shift_idea_minus_baseline": -0.0916648656129837
232 },
233 {
234 "seed": 5,
235 "baseline_mse": 0.004745990503579378,
236 "idea_mse": 0.009611713699996471,
237 "trained_prediction_mean_shift_idea_minus_baseline": 0.023652706295251846
238 },
239 {
240 "seed": 6,
241 "baseline_mse": 0.006178014911711216,
242 "idea_mse": 0.009160290472209454,
243 "trained_prediction_mean_shift_idea_minus_baseline": 0.0069645848125219345
244 },
245 {
246 "seed": 7,
247 "baseline_mse": 0.0056354450061917305,
248 "idea_mse": 0.009701674804091454,
249 "trained_prediction_mean_shift_idea_minus_baseline": -0.05231938883662224
250 }
251 ],
252 "confirmed": false
253 },
254 "custom_track": {
255 "name": "poisson_boundary",
256 "file": "bench/custom_tracks/poisson_boundary.py",
257 "domain": "pde"
258 },
259 "structural_match": "PDE Dirichlet boundary-value regression on the unit disk; WOS is the intervention."
260 }
261}