Adaptive conformal safety margins / results.json

Failed on benchmark

Raw ⬇ ZIP
  1{
  2  "seed": 2068,
  3  "stationary_coverage": [
  4    {
  5      "alpha": 0.1,
  6      "gamma": 0.01,
  7      "predicted_exceedance": 0.1,
  8      "observed_exceedance": 0.09993,
  9      "abs_error": 7.000000000000062e-05
 10    },
 11    {
 12      "alpha": 0.1,
 13      "gamma": 0.03,
 14      "predicted_exceedance": 0.1,
 15      "observed_exceedance": 0.10009,
 16      "abs_error": 8.999999999999286e-05
 17    },
 18    {
 19      "alpha": 0.2,
 20      "gamma": 0.01,
 21      "predicted_exceedance": 0.2,
 22      "observed_exceedance": 0.20007,
 23      "abs_error": 6.999999999998674e-05
 24    },
 25    {
 26      "alpha": 0.2,
 27      "gamma": 0.03,
 28      "predicted_exceedance": 0.2,
 29      "observed_exceedance": 0.20004,
 30      "abs_error": 3.999999999998449e-05
 31    },
 32    {
 33      "alpha": 0.3,
 34      "gamma": 0.01,
 35      "predicted_exceedance": 0.3,
 36      "observed_exceedance": 0.29995,
 37      "abs_error": 4.999999999999449e-05
 38    },
 39    {
 40      "alpha": 0.3,
 41      "gamma": 0.03,
 42      "predicted_exceedance": 0.3,
 43      "observed_exceedance": 0.29997,
 44      "abs_error": 2.999999999997449e-05
 45    },
 46    {
 47      "alpha": 0.4,
 48      "gamma": 0.01,
 49      "predicted_exceedance": 0.4,
 50      "observed_exceedance": 0.39995,
 51      "abs_error": 4.999999999999449e-05
 52    },
 53    {
 54      "alpha": 0.4,
 55      "gamma": 0.03,
 56      "predicted_exceedance": 0.4,
 57      "observed_exceedance": 0.39992,
 58      "abs_error": 8.00000000000245e-05
 59    }
 60  ],
 61  "post_shift_ramp": [
 62    {
 63      "gamma": 0.02,
 64      "predicted_steps": 75,
 65      "observed_steps": 75,
 66      "relative_error": 0.0
 67    },
 68    {
 69      "gamma": 0.04,
 70      "predicted_steps": 38,
 71      "observed_steps": 38,
 72      "relative_error": 0.0
 73    },
 74    {
 75      "gamma": 0.08,
 76      "predicted_steps": 19,
 77      "observed_steps": 19,
 78      "relative_error": 0.0
 79    },
 80    {
 81      "gamma": 0.16,
 82      "predicted_steps": 10,
 83      "observed_steps": 10,
 84      "relative_error": 0.0
 85    }
 86  ],
 87  "recovery_decay": [
 88    {
 89      "gamma": 0.02,
 90      "predicted_steps": 350,
 91      "observed_steps": 350,
 92      "relative_error": 0.0
 93    },
 94    {
 95      "gamma": 0.04,
 96      "predicted_steps": 175,
 97      "observed_steps": 175,
 98      "relative_error": 0.0
 99    },
100    {
101      "gamma": 0.08,
102      "predicted_steps": 88,
103      "observed_steps": 88,
104      "relative_error": 0.0
105    },
106    {
107      "gamma": 0.16,
108      "predicted_steps": 44,
109      "observed_steps": 44,
110      "relative_error": 0.0
111    }
112  ],
113  "crossing_comparison": {
114    "fixed_0.20": {
115      "unsafe_frames": 80,
116      "false_safe_frames": 80,
117      "safe_rejections": 0,
118      "mean_radius": 0.2,
119      "accept_rate": 1.0
120    },
121    "fixed_0.80": {
122      "unsafe_frames": 80,
123      "false_safe_frames": 0,
124      "safe_rejections": 120,
125      "mean_radius": 0.8,
126      "accept_rate": 0.0
127    },
128    "adaptive": {
129      "unsafe_frames": 80,
130      "false_safe_frames": 4,
131      "safe_rejections": 20,
132      "mean_radius": 0.45279999999999954,
133      "accept_rate": 0.52
134    },
135    "quantile_check": {
136      "radii": [
137        0.2,
138        0.4,
139        0.8,
140        1.0
141      ],
142      "beta": 0.25,
143      "upper_quantile": 0.8500000000000001
144    }
145  },
146  "checks": {
147    "stationary_within_0.02": true,
148    "ramp_formula_exact": true,
149    "recovery_formula_exact": true,
150    "mechanism_manifested": true
151  }
152}