Adaptive conformal safety margins / experiment.py

Failed on benchmark

Raw ⬇ ZIP
  1import json, math
  2from pathlib import Path
  3import numpy as np
  4
  5SEED = 2068
  6
  7def update_radius(radius, delta, gamma, alpha):
  8    """Adaptive conformal recursion with the optional nonnegative projection."""
  9    return max(0.0, radius + gamma * (float(delta > radius) - alpha))
 10
 11def conservative_radius(ensemble_radii, beta):
 12    """Empirical upper (1-beta) quantile across estimators."""
 13    return float(np.quantile(np.asarray(ensemble_radii), 1.0 - beta, method="linear"))
 14
 15def stationary_sweep():
 16    rng = np.random.default_rng(SEED)
 17    rows = []
 18    for alpha in (0.10, 0.20, 0.30, 0.40):
 19        for gamma in (0.01, 0.03):
 20            r = 0.5
 21            hits = []
 22            for t in range(120000):
 23                d = rng.exponential(1.0)
 24                hit = d > r
 25                r = update_radius(r, d, gamma, alpha)
 26                if t >= 20000:
 27                    hits.append(hit)
 28            observed = float(np.mean(hits))
 29            rows.append({"alpha": alpha, "gamma": gamma,
 30                         "predicted_exceedance": alpha,
 31                         "observed_exceedance": observed,
 32                         "abs_error": abs(observed - alpha)})
 33    return rows
 34
 35def ramp_sweep():
 36    # If every error exceeds r, dr/dt = gamma(1-alpha).
 37    rows = []
 38    alpha, r0, target = 0.2, 0.2, 1.4
 39    for gamma in (0.02, 0.04, 0.08, 0.16):
 40        r, n = r0, 0
 41        while r < target and n < 10000:
 42            r = update_radius(r, 100.0, gamma, alpha)
 43            n += 1
 44        predicted = math.ceil((target - r0) / (gamma * (1 - alpha)))
 45        rows.append({"gamma": gamma, "predicted_steps": predicted,
 46                     "observed_steps": n,
 47                     "relative_error": abs(n - predicted) / predicted})
 48    return rows
 49
 50def recovery_sweep():
 51    # If every error is below r, dr/dt = -gamma*alpha.
 52    rows = []
 53    alpha, r0, target = 0.2, 1.8, 0.4
 54    for gamma in (0.02, 0.04, 0.08, 0.16):
 55        r, n = r0, 0
 56        while r > target and n < 10000:
 57            r = update_radius(r, 0.0, gamma, alpha)
 58            n += 1
 59        predicted = math.ceil((r0 - target) / (gamma * alpha))
 60        rows.append({"gamma": gamma, "predicted_steps": predicted,
 61                     "observed_steps": n,
 62                     "relative_error": abs(n - predicted) / predicted})
 63    return rows
 64
 65def crossing_comparison():
 66    # Predicted clearance is 1.0 m. A shifted predictor error of .75 m makes
 67    # actual clearance .25 m, below the .55 m physical threshold. Benign error
 68    # is .15 m, giving .85 m actual clearance. Strictly use < for collisions.
 69    alpha, gamma = 0.2, 0.08
 70    errors = np.r_[np.full(60, .15), np.full(80, .75), np.full(60, .15)]
 71    actual_clearance = 1.0 - errors
 72    unsafe = actual_clearance < .55
 73    out = {}
 74    for name, kind in (("fixed_0.20", "fixed"), ("fixed_0.80", "fixed"),
 75                       ("adaptive", "adaptive")):
 76        r = .20
 77        accepted, radii = [], []
 78        for e in errors:
 79            if kind == "adaptive":
 80                radius_used = r
 81                r = update_radius(r, e, gamma, alpha)
 82            else:
 83                radius_used = float(name.split("_")[1])
 84            accepted.append(1.0 > .55 + radius_used)
 85            radii.append(radius_used)
 86        accepted = np.asarray(accepted, dtype=bool)
 87        out[name] = {
 88            "unsafe_frames": int(unsafe.sum()),
 89            "false_safe_frames": int(np.sum(accepted & unsafe)),
 90            "safe_rejections": int(np.sum((~accepted) & (~unsafe))),
 91            "mean_radius": float(np.mean(radii)),
 92            "accept_rate": float(np.mean(accepted))}
 93    # A direct check of the ensemble conservatism formula.
 94    out["quantile_check"] = {"radii": [0.2, 0.4, 0.8, 1.0], "beta": 0.25,
 95                             "upper_quantile": conservative_radius([0.2, 0.4, 0.8, 1.0], 0.25)}
 96    return out
 97
 98def main():
 99    stationary, ramp, recovery = stationary_sweep(), ramp_sweep(), recovery_sweep()
100    crossing = crossing_comparison()
101    coverage_ok = max(x["abs_error"] for x in stationary) < .02
102    ramp_ok = max(x["relative_error"] for x in ramp) == 0.0
103    recovery_ok = max(x["relative_error"] for x in recovery) == 0.0
104    result = {"seed": SEED, "stationary_coverage": stationary,
105              "post_shift_ramp": ramp, "recovery_decay": recovery,
106              "crossing_comparison": crossing,
107              "checks": {"stationary_within_0.02": coverage_ok,
108                         "ramp_formula_exact": ramp_ok,
109                         "recovery_formula_exact": recovery_ok,
110                         "mechanism_manifested": coverage_ok and ramp_ok and recovery_ok}}
111    Path("results.json").write_text(json.dumps(result, indent=2))
112    print(json.dumps(result, indent=2))
113
114if __name__ == "__main__":
115    main()