Стенд-проверка (stage-2) · промт оператора:

(универсальный)

Ответ агента:

{ "worked": false, "confidence": 10, "verdict": "The registered custom graph-decision track budgeted_route_costs was benchmarked end-to-end with matched mlp_tiny systems, a tuned six-configuration baseline sweep, three idea settings, and 8 paired seeds. The adversarial method was significantly worse on independent test MSE (paired delta +0.1220390537, permutation p=0.0081), and its measured worst-case regret was slightly worse, so this is not a win.", "metrics": { "baseline": "test MSE mean 0.0023824744; best lr=0.01, wd=0.0001; worst-regret mean 0.0062907608", "idea": "test MSE mean 0.1244215281; best lr=0.01, wd=0.0001; worst-regret mean 0.0063806495; mechanism confirmed=false" }, "bench_report": { "bench_version": 1, "track": "budgeted_route_costs", "model": "mlp_tiny", "metric_direction": "lower is better", "n_seeds": 8, "baseline": { "best_cfg": { "lr": 0.01, "wd": 0.0001 }, "sweep": [ { "cfg": { "lr": 0.001, "wd": 0.0 }, "mean": 0.024585966 }, { "cfg": { "lr": 0.001, "wd": 0.0001 }, "mean": 0.0238940842 }, { "cfg": { "lr": 0.003, "wd": 0.0 }, "mean": 0.0043507646 }, { "cfg": { "lr": 0.003, "wd": 0.0001 }, "mean": 0.0042273927 }, { "cfg": { "lr": 0.01, "wd": 0.0 }, "mean": 0.002535941 }, { "cfg": { "lr": 0.01, "wd": 0.0001 }, "mean": 0.0022327409 } ], "full": { "mean": 0.0023824744, "std": 0.000329854, "per_seed": [ 0.002391217, 0.0022307509, 0.0022920494, 0.0020169464, 0.0019817667, 0.0030336098, 0.0027303267, 0.002383128 ], "n": 8 } }, "idea": { "mean": 0.1244215281, "std": 0.0674383082, "per_seed": [ 0.0988523215, 0.0738464445, 0.1497047544, 0.2508151531, 0.0538574941, 0.2102787197, 0.0854037479, 0.0726135895 ], "n": 8 }, "comparison": { "delta_mean": 0.1220390537, "idea_wins": 0, "n_pairs": 8, "per_seed_diffs": [ 0.0964611045, 0.0716156936, 0.147412705, 0.2487982067, 0.0518757275, 0.2072451098, 0.0826734211, 0.0702304614 ], "p_value": 0.0081, "mde": 0.060233921, "mde_rel_pct": 2528.2086, "verdict": "idea worse (significant)", "system_worked": false }, "custom_track": { "name": "budgeted_route_costs", "file": "route_track.py", "domain": "graph-decision" }, "idea_sweep": [ { "cfg": { "lr": 0.01, "wd": 0.0001 }, "result": { "mean": 0.1244215281, "std": 0.0674383082, "per_seed": [ 0.0988523215, 0.0738464445, 0.1497047544, 0.2508151531, 0.0538574941, 0.2102787197, 0.0854037479, 0.0726135895 ], "n": 8 } }, { "cfg": { "lr": 0.001, "wd": 0.0001 }, "result": { "mean": 0.2716082148, "std": 0.0501621348, "per_seed": [ 0.327478379, 0.2356351912, 0.3176616728, 0.1626674831, 0.2592439949, 0.2929310203, 0.2722110152, 0.305036962 ], "n": 8 } }, { "cfg": { "lr": 0.01, "wd": 0.0001 }, "result": { "mean": 0.1244215281, "std": 0.0674383082, "per_seed": [ 0.0988523215, 0.0738464445, 0.1497047544, 0.2508151531, 0.0538574941, 0.2102787197, 0.0854037479, 0.0726135895 ], "n": 8 } } ], "mechanism_signature": { "baseline_worst_regret_mean": 0.0062907608, "idea_worst_regret_mean": 0.0063806495, "predicted_effect": "adversarial training reduces budgeted path flips/regret", "confirmed": false } }, "how_to_run": "/home/maxwelhelp/main/bin/python3 stage2_bench.py", "files": [ "route_track.py", "stage2_bench.py", "bench_report.json" ], "limitations": "The built-in tracks lack shortest-path graph structure, so the accepted custom two-parallel-route track was used. It tests four edges and budget B=1, not larger random graphs or road networks; the training surrogate is a soft route expectation over five enumerated scenarios, with exact hard regret evaluation.", "system_verdict": "failed", "practical_verdict": "harms", "mechanism_ok": 0, "system_judged": true }