1{
 2  "idea": "Conformal Early-Rejection for Diffusion Architecture Search",
 3  "mechanism_check": {
 4    "prediction_1": {
 5      "claim": "For iid continuous checkpoint scores, the empirical split-conformal threshold has fresh-score exceedance approximately (n-k+1)/(n+1), at or below alpha=0.10.",
 6      "observed_vs_predicted": [
 7        {"ncal": 20, "predicted": 0.09524, "observed": 0.09381},
 8        {"ncal": 50, "predicted": 0.09804, "observed": 0.09930},
 9        {"ncal": 100, "predicted": 0.09901, "observed": 0.09891},
10        {"ncal": 500, "predicted": 0.09980, "observed": 0.09994}
11      ],
12      "status": "confirmed"
13    },
14    "prediction_2": {
15      "claim": "If the monitor is useful, increasing score separability should materially increase rejection of failures and lower accepted-candidate failure while retaining useful candidates.",
16      "sweep": [
17        {"beta": 0.5, "rejection_rate": 0.1648, "accepted_failure": 0.3998},
18        {"beta": 1.0, "rejection_rate": 0.1695, "accepted_failure": 0.3980},
19        {"beta": 2.0, "rejection_rate": 0.1619, "accepted_failure": 0.4014},
20        {"beta": 5.0, "rejection_rate": 0.1639, "accepted_failure": 0.4032},
21        {"beta": 10.0, "rejection_rate": 0.1690, "accepted_failure": 0.3963}
22      ],
23      "status": "not_confirmed; no monotonic or material effect"
24    },
25    "prediction_3": {
26      "claim": "Exchangeability shift should degrade the accepted-failure guarantee.",
27      "observed": [
28        {"test_shift": 0.0, "rejection_rate": 0.1639, "accepted_failure": 0.3992},
29        {"test_shift": -0.25, "rejection_rate": 0.2249, "accepted_failure": 0.4799},
30        {"test_shift": -0.5, "rejection_rate": 0.3035, "accepted_failure": 0.5535},
31        {"test_shift": -1.0, "rejection_rate": 0.4888, "accepted_failure": 0.6873}
32      ],
33      "status": "confirmed as degradation, but iid accepted failure is already far above alpha"
34    }
35  },
36  "controller_comparison": {
37    "alpha_0.10": {
38      "none": {"full_evaluations_per_1000": 1000.0, "accepted_failure": 0.5025},
39      "conformal": {"full_evaluations_per_1000": 828.75, "compute_saved": 0.1713, "accepted_failure": 0.3998},
40      "uncalibrated": {"full_evaluations_per_1000": 357.55, "compute_saved": 0.6425, "accepted_failure": 0.0213}
41    },
42    "alpha_0.05": {
43      "none": {"full_evaluations_per_1000": 1000.0, "accepted_failure": 0.5025},
44      "conformal": {"full_evaluations_per_1000": 905.14, "compute_saved": 0.0949, "accepted_failure": 0.4503},
45      "uncalibrated": {"full_evaluations_per_1000": 357.55, "compute_saved": 0.6425, "accepted_failure": 0.0213}
46    }
47  },
48  "interpretation": "The order-statistic formula works for marginal score coverage, but the proposed implication Pr(failure | continue) <= alpha does not follow from an upper quantile of failure-risk scores. Rejecting only the highest 10% of scores leaves many failures among accepted candidates. Distribution shift worsens this substantially. The synthetic experiment therefore does not demonstrate the promised mechanism-level safety/efficiency tradeoff.",
49  "reproduce": "/home/maxwelhelp/main/bin/python3 conformal_early_rejection.py",
50  "seed": 2762
51}