Стенд-проверка (stage-2) · промт оператора:

(универсальный)

Ответ агента:

{ "worked": false, "confidence": 9, "verdict": "Implemented a dynamics-matched feasible-action projection layer around independently trained rnn_small systems, with equal-budget baseline and idea sweeps, 8 paired seeds, and permutation testing. The trained-model mechanism signature was confirmed: projection distance was zero for a subset of predictions and positive outside the reachable interval. However, the safety-layer system had substantially higher test MSE (0.0078135 vs 0.0007652), with paired delta +0.0070482 and p=0.0081, so there was no benchmark win.", "metrics": { "baseline": "Test MSE mean 0.0007652444473933429, std 0.00014266442888030616; best config lr=0.006, epochs=18.", "idea": "Test MSE mean 0.00781346601434052, std 0.0006524094833102025; best tested config lr=0.001, epochs=18. Paired delta +0.007048221566947177, p=0.0081, 0/8 idea wins.", "mechanism_signature": "observed_inside_fraction=0.1750, mean_projection_distance=0.0703845, max_projection_distance=0.286004, confirmed=true" }, "how_to_run": "/home/maxwelhelp/main/bin/python3 stage2_bench.py", "files": [ "stage2_bench.py", "bench_report.json" ], "limitations": "The built-in dynamics bench is supervised future-angle prediction rather than closed-loop PPO or actuator execution, so recursive feasibility, solver failures, model mismatch, backup control, and deployment runtime were not tested. The projection uses a conservative scalar bounded-acceleration reachability approximation for the scalar target rather than a full nonlinear pendulum OCP.", "bench_report": { "bench_version": 1, "track": "dynamics", "model": "rnn_small", "metric_direction": "lower is better", "n_seeds": 8, "baseline": { "best_cfg": { "lr": 0.006, "epochs": 18 }, "sweep": [ { "cfg": { "lr": 0.001, "epochs": 12 }, "mean": 0.004545392468571663 }, { "cfg": { "lr": 0.001, "epochs": 18 }, "mean": 0.001921573159052059 }, { "cfg": { "lr": 0.003, "epochs": 12 }, "mean": 0.00358188062091358 }, { "cfg": { "lr": 0.003, "epochs": 18 }, "mean": 0.0011179699067724869 }, { "cfg": { "lr": 0.006, "epochs": 12 }, "mean": 0.0019017510931007564 }, { "cfg": { "lr": 0.006, "epochs": 18 }, "mean": 0.0007884320366429165 } ], "full": { "mean": 0.0007652444473933429, "std": 0.00014266442888030616, "per_seed": [ 0.0007294044480659068, 0.0007733436650596559, 0.0005735447048209608, 0.0010774353286251426, 0.0006340121617540717, 0.0008162351441569626, 0.0006983152125030756, 0.0008196649141609669 ], "n": 8 } }, "idea": { "mean": 0.00781346601434052, "std": 0.0006524094833102025, "per_seed": [ 0.00748194707557559, 0.008646145462989807, 0.007898174226284027, 0.008135655894875526, 0.006896249484270811, 0.006903830915689468, 0.008720104582607746, 0.007825620472431183 ], "n": 8 }, "comparison": { "delta_mean": 0.007048221566947177, "idea_wins": 0, "n_pairs": 8, "per_seed_diffs": [ 0.006752542627509683, 0.007872801797930151, 0.007324629521463066, 0.007058220566250384, 0.006262237322516739, 0.006087595771532506, 0.00802178937010467, 0.007005955558270216 ], "p_value": 0.0081, "mde": 0.0005770632681359283, "mde_rel_pct": 75.40901082021342, "verdict": "idea worse (significant)", "system_worked": false }, "mechanism_signature": { "prediction": "feasible predictions have zero projection distance; infeasible predictions have positive distance", "observed_inside_fraction": 0.17499999701976776, "observed_mean_projection_distance": 0.07038450986146927, "observed_max_projection_distance": 0.2860039472579956, "confirmed": true, "idea_sweep": [ { "cfg": { "lr": 0.006, "epochs": 18 }, "mean": 0.00799608719535172 }, { "cfg": { "lr": 0.003, "epochs": 18 }, "mean": 0.007877308991737664 }, { "cfg": { "lr": 0.001, "epochs": 18 }, "mean": 0.00781346601434052 } ], "budget": { "epochs": 18, "batch": 128 } } }, "system_verdict": "partial", "practical_verdict": "harms", "mechanism_ok": 1, "system_judged": true }