Стенд-проверка (stage-2) · промт оператора:

(универсальный)

Ответ агента:

{ "worked": false, "confidence": 10, "verdict": "The reachable-set risk head was implemented on the registered dynamics/rnn_small benchmark with a matched GRU backbone and an added Gaussian risk-training loss. It significantly worsened standard test MSE relative to the tuned baseline: paired delta +1.42697 with permutation p=0.0081, so the benchmark verdict is idea worse (significant). The trained-model mechanism signature was also not confirmed quantitatively.", "metrics": { "baseline": "dynamics/rnn_small, best lr=0.006, full 8-seed test MSE mean 0.002105819425196387", "idea": "reachable-risk GRU, best lr=0.001 and risk_weight=0.08, full 8-seed test MSE mean 1.4290764033794403" }, "how_to_run": "/home/maxwelhelp/main/bin/python3 bench_reachable.py", "files": [ "bench_reachable.py", "bench_report.json" ], "limitations": "The built-in dynamics benchmark is one-step pendulum regression, so the implementation used a next-state safety margin and finite-horizon union transform rather than full multi-step autograd Jacobian covariance propagation. Calibration fitting, temperature scaling, ensembles, multiple half-spaces, and long-horizon warning recall were not tested. The custom idea path fell back to CPU after a shared-GPU allocation failure.", "bench_report": { "bench_version": 1, "track": "dynamics", "model": "rnn_small", "metric_direction": "lower is better", "n_seeds": 8, "baseline": { "best_cfg": { "lr": 0.006 }, "sweep": [ { "cfg": { "lr": 0.001 }, "mean": 0.004298779065720737 }, { "cfg": { "lr": 0.003 }, "mean": 0.0034615940821822733 }, { "cfg": { "lr": 0.006 }, "mean": 0.0018282315868418664 } ], "full": { "mean": 0.002105819425196387, "std": 0.000389436566226016, "per_seed": [ 0.002004395006224513, 0.001437882543541491, 0.0015748648438602686, 0.002295783953741193, 0.0024020641576498747, 0.0024898042902350426, 0.0020851334556937218, 0.0025566271506249905 ], "n": 8 } }, "idea": { "mean": 1.4290764033794403, "std": 0.10048076612287149, "per_seed": [ 1.3990658521652222, 1.322527527809143, 1.6020216941833496, 1.576513409614563, 1.3529068231582642, 1.4524457454681396, 1.3324311971664429, 1.394698977470398 ], "n": 8, "sweep": [ { "cfg": { "lr": 0.006, "risk_weight": 0.15 }, "mean": 1.5168477594852448 }, { "cfg": { "lr": 0.001, "risk_weight": 0.08 }, "mean": 1.4750321209430695 }, { "cfg": { "lr": 0.006, "risk_weight": 0.25 }, "mean": 1.4902327060699463 } ], "best_cfg": { "lr": 0.001, "risk_weight": 0.08 } }, "comparison": { "delta_mean": 1.426970583954244, "idea_wins": 0, "n_pairs": 8, "per_seed_diffs": [ 1.3970614571589977, 1.3210896452656016, 1.6004468293394893, 1.5742176256608218, 1.3505047590006143, 1.4499559411779046, 1.3303460637107491, 1.392142350319773 ], "p_value": 0.0081, "mde": 0.08983758701469045, "mde_rel_pct": 4266.1581491638235, "verdict": "idea worse (significant)", "system_worked": false }, "mechanism_signature": { "predicted": "risk increases as Mahalanobis safety margin decreases", "margin_risk_correlation": -0.5783055626522379, "low_margin_risk": 0.3624718189239502, "high_margin_risk": 0.0, "confirmed": false }, "protocol_notes": "Paired 8 seeds; baseline and idea share GRU width, epochs, batch and learning-rate union; idea modifies training with Gaussian risk loss." }, "system_verdict": "failed", "practical_verdict": "harms", "mechanism_ok": 0, "system_judged": true }