Spatial-Quantile Conformal Bands for Neural Operators / bench_report.json
Mechanism confirmed, baseline not beaten
1{
2 "bench_version": 1,
3 "track": "poisson_field_operator",
4 "model": "mlp_tiny",
5 "metric_direction": "lower is better",
6 "n_seeds": 8,
7 "baseline": {
8 "best_cfg": {
9 "lr": 0.006
10 },
11 "sweep": [
12 {
13 "cfg": {
14 "lr": 0.0015
15 },
16 "mean": 0.004578319028951228
17 },
18 {
19 "cfg": {
20 "lr": 0.003
21 },
22 "mean": 0.0021311483287718147
23 },
24 {
25 "cfg": {
26 "lr": 0.006
27 },
28 "mean": 0.0014657231222372502
29 }
30 ],
31 "full": {
32 "mean": 0.0014413107419386506,
33 "std": 8.870037700098626e-05,
34 "per_seed": [
35 0.0013521382352337241,
36 0.0014695348218083382,
37 0.001459407852962613,
38 0.0015818115789443254,
39 0.0015570472460240126,
40 0.0013796440325677395,
41 0.0013133999891579151,
42 0.0014175021788105369
43 ],
44 "n": 8
45 }
46 },
47 "idea": {
48 "mean": 0.0014413107419386506,
49 "std": 8.870037700098626e-05,
50 "per_seed": [
51 0.0013521382352337241,
52 0.0014695348218083382,
53 0.001459407852962613,
54 0.0015818115789443254,
55 0.0015570472460240126,
56 0.0013796440325677395,
57 0.0013133999891579151,
58 0.0014175021788105369
59 ],
60 "n": 8
61 },
62 "comparison": {
63 "delta_mean": 0.0,
64 "idea_wins": 0,
65 "n_pairs": 8,
66 "per_seed_diffs": [
67 0.0,
68 0.0,
69 0.0,
70 0.0,
71 0.0,
72 0.0,
73 0.0,
74 0.0
75 ],
76 "p_value": 1.0,
77 "mde": 0.0,
78 "mde_rel_pct": 0.0,
79 "verdict": "no measurable effect",
80 "system_worked": false
81 },
82 "mechanism_signature": {
83 "mechanism_signature": {
84 "prediction": "spatial quantile should reduce mean band width for gamma=.10 while retaining approximately 90% domain-event rate",
85 "observed_mean_max_width": 0.24931471447160236,
86 "observed_mean_quantile_width": 0.15621099945286587,
87 "observed_mean_max_event_rate": 0.9962500000000001,
88 "observed_mean_quantile_event_rate": 0.8981250000000001,
89 "confirmed": true,
90 "per_seed": [
91 {
92 "seed": 0,
93 "max_width": 0.23654802549230663,
94 "quantile_width": 0.14455619026329847,
95 "max_event_rate": 1.0,
96 "quantile_event_rate": 0.875
97 },
98 {
99 "seed": 1,
100 "max_width": 0.2480752385754918,
101 "quantile_width": 0.15798166727590549,
102 "max_event_rate": 0.995,
103 "quantile_event_rate": 0.89
104 },
105 {
106 "seed": 2,
107 "max_width": 0.2506383314002877,
108 "quantile_width": 0.15749358108192799,
109 "max_event_rate": 0.995,
110 "quantile_event_rate": 0.88
111 },
112 {
113 "seed": 3,
114 "max_width": 0.2323479681994933,
115 "quantile_width": 0.15607988602021236,
116 "max_event_rate": 0.995,
117 "quantile_event_rate": 0.885
118 },
119 {
120 "seed": 4,
121 "max_width": 0.2446657397553012,
122 "quantile_width": 0.1554601836897333,
123 "max_event_rate": 1.0,
124 "quantile_event_rate": 0.87
125 },
126 {
127 "seed": 5,
128 "max_width": 0.26013515596171516,
129 "quantile_width": 0.15499080343415642,
130 "max_event_rate": 1.0,
131 "quantile_event_rate": 0.92
132 },
133 {
134 "seed": 6,
135 "max_width": 0.24530170790579806,
136 "quantile_width": 0.1605553398963065,
137 "max_event_rate": 0.985,
138 "quantile_event_rate": 0.945
139 },
140 {
141 "seed": 7,
142 "max_width": 0.27680554848242506,
143 "quantile_width": 0.16257034396138648,
144 "max_event_rate": 1.0,
145 "quantile_event_rate": 0.92
146 }
147 ]
148 },
149 "idea_sweep": [
150 {
151 "cfg": {
152 "lr": 0.0015,
153 "gamma": 0.1
154 },
155 "result": {
156 "mean": 0.004449288360774517,
157 "std": 0.00023043464415988465,
158 "per_seed": [
159 0.004382778890430927,
160 0.004531833343207836,
161 0.004430715460330248,
162 0.004967948421835899,
163 0.004390249960124493,
164 0.0041168006137013435,
165 0.004492855165153742,
166 0.004281125031411648
167 ],
168 "n": 8
169 }
170 },
171 {
172 "cfg": {
173 "lr": 0.003,
174 "gamma": 0.1
175 },
176 "result": {
177 "mean": 0.0021061655279481784,
178 "std": 0.0001495587726895796,
179 "per_seed": [
180 0.00186435307841748,
181 0.0021284606773406267,
182 0.0021266459953039885,
183 0.0024051335640251637,
184 0.002195546869188547,
185 0.0021005580201745033,
186 0.0020656778942793608,
187 0.0019629481248557568
188 ],
189 "n": 8
190 }
191 },
192 {
193 "cfg": {
194 "lr": 0.006,
195 "gamma": 0.1
196 },
197 "result": {
198 "mean": 0.0014413107419386506,
199 "std": 8.870037700098626e-05,
200 "per_seed": [
201 0.0013521382352337241,
202 0.0014695348218083382,
203 0.001459407852962613,
204 0.0015818115789443254,
205 0.0015570472460240126,
206 0.0013796440325677395,
207 0.0013133999891579151,
208 0.0014175021788105369
209 ],
210 "n": 8
211 }
212 }
213 ],
214 "custom_track": {
215 "name": "poisson_field_operator",
216 "file": "poisson_field_track.py",
217 "domain": "pde"
218 }
219 }
220}