Стенд-проверка (stage-2) · промт оператора:

(универсальный)

Ответ агента:

{ "worked": false, "confidence": 9, "verdict": "Built and ran a structurally matched custom OBB trajectory-regression track with identical MLP systems, paired 8-seed evaluation, equal learning-rate sweeps, differentiable SAT barrier training, and a trained-model clearance signature. The barrier reduced observed collision fraction to 0.3026 and improved predicted margins, but test MSE increased from 0.11758 to 0.19739; paired delta was +0.07982 with permutation p=0.02415. The safety mechanism is directionally present, but the idea is significantly worse on the standard task metric and is not a benchmark win.", "metrics": { "baseline": "MSE 0.1175788217 mean, std 0.0498898437, best lr 0.01, 8 seeds", "idea": "MSE 0.1973940274 mean, std 0.1311616229, best lr 0.01, lambda 0.08, tau 0.1, m0 0.05, beta 0.05; collision fraction 0.3025781" }, "bench_report": { "bench_version": 1, "track": "obb_clearance_trajectory", "model": "mlp_tiny", "metric_direction": "lower is better", "n_seeds": 8, "baseline": { "best_cfg": { "lr": 0.01, "epochs": 24 }, "sweep": [ { "cfg": { "lr": 0.001, "epochs": 24 }, "mean": 2.5900412798 }, { "cfg": { "lr": 0.003, "epochs": 24 }, "mean": 1.0762107968 }, { "cfg": { "lr": 0.01, "epochs": 24 }, "mean": 0.0882154824 } ], "full": { "mean": 0.1175788217, "std": 0.0498898437, "per_seed": [ 0.1170619577, 0.1304021031, 0.0387990363, 0.0665988326, 0.1332305372, 0.2189174742, 0.1324296296, 0.1031910032 ], "n": 8 } }, "idea": { "mean": 0.1973940274, "std": 0.1311616229, "per_seed": [ 0.1260967106, 0.1376173794, 0.1566873342, 0.1245219335, 0.2037198246, 0.537575841, 0.1235302687, 0.1694029272 ], "n": 8 }, "comparison": { "delta_mean": 0.0798152057, "idea_wins": 1, "n_pairs": 8, "per_seed_diffs": [ 0.0090347528, 0.0072152764, 0.1178882979, 0.0579231009, 0.0704892874, 0.3186583668, -0.0088993609, 0.06621192396 ], "p_value": 0.02415, "verdict": "idea worse (significant)", "system_worked": false }, "mechanism_signature": { "math_check": { "predicted_exact_zero_m": 1.1, "observed_exact_zero_m": 1.1, "observed_smooth_zero_m": 1.03, "smoothing_bound_holds": true }, "predicted_vs_observed": { "predicted_smoothed_margin_mean": 0.2649075, "observed_exact_sat_margin_mean": -0.4316907, "observed_collision_fraction": 0.3025781 }, "prediction": "barrier should reduce collision fraction and move margins upward", "confirmed": true }, "custom_track": { "name": "obb_clearance_trajectory", "file": "obb_track.py", "domain": "geometric_collision_avoidance" } }, "how_to_run": "/home/maxwelhelp/main/bin/python3 obb_bench.py", "files": [ "obb_track.py", "obb_bench.py", "bench_report.json" ], "limitations": "The custom track is synthetic rather than nuScenes/nuPlan and uses a small MLP and 24 epochs. Exact polygon clearance, multiple agents, inference reranking, FLOP-normalized comparisons, and broader barrier-weight sweeps were not tested.", "system_verdict": "partial", "practical_verdict": "harms", "mechanism_ok": 1, "system_judged": true }