{
  "idea": "Conformal Early-Rejection for Diffusion Architecture Search",
  "mechanism_check": {
    "prediction_1": {
      "claim": "For iid continuous checkpoint scores, the empirical split-conformal threshold has fresh-score exceedance approximately (n-k+1)/(n+1), at or below alpha=0.10.",
      "observed_vs_predicted": [
        {"ncal": 20, "predicted": 0.09524, "observed": 0.09381},
        {"ncal": 50, "predicted": 0.09804, "observed": 0.09930},
        {"ncal": 100, "predicted": 0.09901, "observed": 0.09891},
        {"ncal": 500, "predicted": 0.09980, "observed": 0.09994}
      ],
      "status": "confirmed"
    },
    "prediction_2": {
      "claim": "If the monitor is useful, increasing score separability should materially increase rejection of failures and lower accepted-candidate failure while retaining useful candidates.",
      "sweep": [
        {"beta": 0.5, "rejection_rate": 0.1648, "accepted_failure": 0.3998},
        {"beta": 1.0, "rejection_rate": 0.1695, "accepted_failure": 0.3980},
        {"beta": 2.0, "rejection_rate": 0.1619, "accepted_failure": 0.4014},
        {"beta": 5.0, "rejection_rate": 0.1639, "accepted_failure": 0.4032},
        {"beta": 10.0, "rejection_rate": 0.1690, "accepted_failure": 0.3963}
      ],
      "status": "not_confirmed; no monotonic or material effect"
    },
    "prediction_3": {
      "claim": "Exchangeability shift should degrade the accepted-failure guarantee.",
      "observed": [
        {"test_shift": 0.0, "rejection_rate": 0.1639, "accepted_failure": 0.3992},
        {"test_shift": -0.25, "rejection_rate": 0.2249, "accepted_failure": 0.4799},
        {"test_shift": -0.5, "rejection_rate": 0.3035, "accepted_failure": 0.5535},
        {"test_shift": -1.0, "rejection_rate": 0.4888, "accepted_failure": 0.6873}
      ],
      "status": "confirmed as degradation, but iid accepted failure is already far above alpha"
    }
  },
  "controller_comparison": {
    "alpha_0.10": {
      "none": {"full_evaluations_per_1000": 1000.0, "accepted_failure": 0.5025},
      "conformal": {"full_evaluations_per_1000": 828.75, "compute_saved": 0.1713, "accepted_failure": 0.3998},
      "uncalibrated": {"full_evaluations_per_1000": 357.55, "compute_saved": 0.6425, "accepted_failure": 0.0213}
    },
    "alpha_0.05": {
      "none": {"full_evaluations_per_1000": 1000.0, "accepted_failure": 0.5025},
      "conformal": {"full_evaluations_per_1000": 905.14, "compute_saved": 0.0949, "accepted_failure": 0.4503},
      "uncalibrated": {"full_evaluations_per_1000": 357.55, "compute_saved": 0.6425, "accepted_failure": 0.0213}
    }
  },
  "interpretation": "The order-statistic formula works for marginal score coverage, but the proposed implication Pr(failure | continue) <= alpha does not follow from an upper quantile of failure-risk scores. Rejecting only the highest 10% of scores leaves many failures among accepted candidates. Distribution shift worsens this substantially. The synthetic experiment therefore does not demonstrate the promised mechanism-level safety/efficiency tradeoff.",
  "reproduce": "/home/maxwelhelp/main/bin/python3 conformal_early_rejection.py",
  "seed": 2762
}