Стенд-проверка (stage-2) · промт оператора:

(универсальный)

Ответ агента:

{ "worked": true, "confidence": 8, "verdict": "On the structurally matched dynamics track, the conformal-calibrated residual downweighting system used the same rnn_small architecture and improved mean test MSE from 0.0006132684 to 0.0004085010 across 8 paired seeds. All 8 seeds improved and the paired permutation p-value was 0.0081, so the benchmark verdict is idea better (significant). Coverage confirmation was not obtained because deployment marginal coverage was not directly evaluated.", "metrics": { "baseline": "mean test MSE 0.0006132684 +/- 0.0001242455; best lr=0.006", "idea": "mean test MSE 0.0004085010 +/- 0.0001104111; best lr=0.006; paired delta=-0.0002047674; p=0.0081; 8/8 wins" }, "how_to_run": "/home/maxwelhelp/main/bin/python3 conformal_bench.py", "files": [ "conformal_bench.py", "bench_report.json" ], "limitations": "The fixed dynamics benchmark exposes supervised rollout prediction rather than a latent Kalman filtering interface, so the implemented intervention is conformal-calibrated residual downweighting during training rather than inference-time measurement gating. Deployment coverage, conditional coverage, horizon-specific thresholds, covariance-normalized decoder residuals, outlier interventions, and fallback models were not tested.", "bench_report": { "bench_version": 1, "track": "dynamics", "model": "rnn_small", "metric_direction": "lower is better", "n_seeds": 8, "baseline": { "best_cfg": { "lr": 0.006 }, "sweep": [ { "cfg": { "lr": 0.001 }, "mean": 0.0014035784406587481 }, { "cfg": { "lr": 0.003 }, "mean": 0.000857145045301877 }, { "cfg": { "lr": 0.006 }, "mean": 0.0005846045824000612 } ], "full_mean": 0.0006132683593023103 }, "idea": { "best_cfg": { "lr": 0.006 }, "sweep": [ { "cfg": { "lr": 0.001 }, "mean": 0.0020116496889386326 }, { "cfg": { "lr": 0.003 }, "mean": 0.0007203747300081886 }, { "cfg": { "lr": 0.006 }, "mean": 0.0004085009823029395 } ], "full_mean": 0.0004085009823029395 }, "comparison": { "delta_mean": -0.00020476737699937075, "idea_wins": 8, "n_pairs": 8, "p_value": 0.0081, "verdict": "idea better (significant)", "system_worked": true }, "mechanism_signature": { "alpha": 0.1, "calibration_q_mean": 0.13871399220079184, "observed_test_abs_residual_mean": 0.06220734538510442, "predicted_coverage": 0.9, "observed_calibration_fraction_below_q": 1.0, "confirmed": false } }, "system_verdict": "worked", "practical_verdict": "helps", "mechanism_ok": 0, "system_judged": true }