Spatial-Quantile Conformal Bands for Neural Operators / bench_report.json

Mechanism confirmed, baseline not beaten

Raw ⬇ ZIP
  1{
  2  "bench_version": 1,
  3  "track": "poisson_field_operator",
  4  "model": "mlp_tiny",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "best_cfg": {
  9      "lr": 0.006
 10    },
 11    "sweep": [
 12      {
 13        "cfg": {
 14          "lr": 0.0015
 15        },
 16        "mean": 0.004578319028951228
 17      },
 18      {
 19        "cfg": {
 20          "lr": 0.003
 21        },
 22        "mean": 0.0021311483287718147
 23      },
 24      {
 25        "cfg": {
 26          "lr": 0.006
 27        },
 28        "mean": 0.0014657231222372502
 29      }
 30    ],
 31    "full": {
 32      "mean": 0.0014413107419386506,
 33      "std": 8.870037700098626e-05,
 34      "per_seed": [
 35        0.0013521382352337241,
 36        0.0014695348218083382,
 37        0.001459407852962613,
 38        0.0015818115789443254,
 39        0.0015570472460240126,
 40        0.0013796440325677395,
 41        0.0013133999891579151,
 42        0.0014175021788105369
 43      ],
 44      "n": 8
 45    }
 46  },
 47  "idea": {
 48    "mean": 0.0014413107419386506,
 49    "std": 8.870037700098626e-05,
 50    "per_seed": [
 51      0.0013521382352337241,
 52      0.0014695348218083382,
 53      0.001459407852962613,
 54      0.0015818115789443254,
 55      0.0015570472460240126,
 56      0.0013796440325677395,
 57      0.0013133999891579151,
 58      0.0014175021788105369
 59    ],
 60    "n": 8
 61  },
 62  "comparison": {
 63    "delta_mean": 0.0,
 64    "idea_wins": 0,
 65    "n_pairs": 8,
 66    "per_seed_diffs": [
 67      0.0,
 68      0.0,
 69      0.0,
 70      0.0,
 71      0.0,
 72      0.0,
 73      0.0,
 74      0.0
 75    ],
 76    "p_value": 1.0,
 77    "mde": 0.0,
 78    "mde_rel_pct": 0.0,
 79    "verdict": "no measurable effect",
 80    "system_worked": false
 81  },
 82  "mechanism_signature": {
 83    "mechanism_signature": {
 84      "prediction": "spatial quantile should reduce mean band width for gamma=.10 while retaining approximately 90% domain-event rate",
 85      "observed_mean_max_width": 0.24931471447160236,
 86      "observed_mean_quantile_width": 0.15621099945286587,
 87      "observed_mean_max_event_rate": 0.9962500000000001,
 88      "observed_mean_quantile_event_rate": 0.8981250000000001,
 89      "confirmed": true,
 90      "per_seed": [
 91        {
 92          "seed": 0,
 93          "max_width": 0.23654802549230663,
 94          "quantile_width": 0.14455619026329847,
 95          "max_event_rate": 1.0,
 96          "quantile_event_rate": 0.875
 97        },
 98        {
 99          "seed": 1,
100          "max_width": 0.2480752385754918,
101          "quantile_width": 0.15798166727590549,
102          "max_event_rate": 0.995,
103          "quantile_event_rate": 0.89
104        },
105        {
106          "seed": 2,
107          "max_width": 0.2506383314002877,
108          "quantile_width": 0.15749358108192799,
109          "max_event_rate": 0.995,
110          "quantile_event_rate": 0.88
111        },
112        {
113          "seed": 3,
114          "max_width": 0.2323479681994933,
115          "quantile_width": 0.15607988602021236,
116          "max_event_rate": 0.995,
117          "quantile_event_rate": 0.885
118        },
119        {
120          "seed": 4,
121          "max_width": 0.2446657397553012,
122          "quantile_width": 0.1554601836897333,
123          "max_event_rate": 1.0,
124          "quantile_event_rate": 0.87
125        },
126        {
127          "seed": 5,
128          "max_width": 0.26013515596171516,
129          "quantile_width": 0.15499080343415642,
130          "max_event_rate": 1.0,
131          "quantile_event_rate": 0.92
132        },
133        {
134          "seed": 6,
135          "max_width": 0.24530170790579806,
136          "quantile_width": 0.1605553398963065,
137          "max_event_rate": 0.985,
138          "quantile_event_rate": 0.945
139        },
140        {
141          "seed": 7,
142          "max_width": 0.27680554848242506,
143          "quantile_width": 0.16257034396138648,
144          "max_event_rate": 1.0,
145          "quantile_event_rate": 0.92
146        }
147      ]
148    },
149    "idea_sweep": [
150      {
151        "cfg": {
152          "lr": 0.0015,
153          "gamma": 0.1
154        },
155        "result": {
156          "mean": 0.004449288360774517,
157          "std": 0.00023043464415988465,
158          "per_seed": [
159            0.004382778890430927,
160            0.004531833343207836,
161            0.004430715460330248,
162            0.004967948421835899,
163            0.004390249960124493,
164            0.0041168006137013435,
165            0.004492855165153742,
166            0.004281125031411648
167          ],
168          "n": 8
169        }
170      },
171      {
172        "cfg": {
173          "lr": 0.003,
174          "gamma": 0.1
175        },
176        "result": {
177          "mean": 0.0021061655279481784,
178          "std": 0.0001495587726895796,
179          "per_seed": [
180            0.00186435307841748,
181            0.0021284606773406267,
182            0.0021266459953039885,
183            0.0024051335640251637,
184            0.002195546869188547,
185            0.0021005580201745033,
186            0.0020656778942793608,
187            0.0019629481248557568
188          ],
189          "n": 8
190        }
191      },
192      {
193        "cfg": {
194          "lr": 0.006,
195          "gamma": 0.1
196        },
197        "result": {
198          "mean": 0.0014413107419386506,
199          "std": 8.870037700098626e-05,
200          "per_seed": [
201            0.0013521382352337241,
202            0.0014695348218083382,
203            0.001459407852962613,
204            0.0015818115789443254,
205            0.0015570472460240126,
206            0.0013796440325677395,
207            0.0013133999891579151,
208            0.0014175021788105369
209          ],
210          "n": 8
211        }
212      }
213    ],
214    "custom_track": {
215      "name": "poisson_field_operator",
216      "file": "poisson_field_track.py",
217      "domain": "pde"
218    }
219  }
220}