Стенд-проверка (stage-2) · промт оператора:

(универсальный)

Ответ агента:

{ "worked": false, "confidence": 9, "verdict": "Implemented Collision-Aware Subset Attention as a local exact subset-posterior mechanism over four-token neighborhoods inside a matched sequence transformer, using the canonical benchmark training path. The mechanism prediction held quantitatively: boosting pair compatibility increased trained-model joint inclusion mass from 0.0397 to 0.2282. However, test MSE was significantly worse than the standard attention baseline (0.23773 vs 0.16653, paired delta +0.07121, permutation p=0.0081), so there is no demonstrated transfer win on the standard sequence benchmark.", "metrics": { "baseline": "test MSE mean=0.166526, std=0.019344; best lr=0.001; per-seed=[0.189505,0.150417,0.141310,0.178840,0.151882,0.159856,0.200021,0.160381]", "idea": "test MSE mean=0.237731, std=0.022670; best lr=0.001; per-seed=[0.242086,0.213451,0.242339,0.202913,0.272864,0.231953,0.267943,0.228303]", "paired_comparison": "delta_mean=+0.071205 (idea-baseline; lower is better), idea_wins=0/8, p_value=0.0081, verdict='idea worse (significant)'" }, "bench_report": { "bench_version": 1, "track": "sequence", "model": "transformer_tiny", "metric_direction": "lower is better", "n_seeds": 8, "baseline": { "best_cfg": { "lr": 0.001, "epochs": 12 }, "sweep": [ { "cfg": { "lr": 0.001, "epochs": 12 }, "mean": 0.16501786187291145 }, { "cfg": { "lr": 0.003, "epochs": 12 }, "mean": 0.1727909930050373 }, { "cfg": { "lr": 0.006, "epochs": 12 }, "mean": 0.17529388144612312 } ], "full": { "mean": 0.16652636602520943, "std": 0.019344256210385904, "per_seed": [ 0.18950511515140533, 0.15041682124137878, 0.14130990207195282, 0.17883960902690887, 0.15188206732273102, 0.1598561406135559, 0.20002053678035736, 0.16038073599338531 ], "n": 8 } }, "idea": { "mean": 0.23773143999278545, "std": 0.022669798148183287, "per_seed": [ 0.24208609759807587, 0.21345070004463196, 0.24233895540237427, 0.20291335880756378, 0.27286380529403687, 0.23195308446884155, 0.26794296503067017, 0.22830255329608917 ], "n": 8, "best_cfg": { "lr": 0.001, "epochs": 12 }, "sweep": [ { "cfg": { "lr": 0.001, "epochs": 12 }, "mean": 0.23773143999278545 }, { "cfg": { "lr": 0.003, "epochs": 12 }, "mean": 0.25968209840357304 }, { "cfg": { "lr": 0.006, "epochs": 12 }, "mean": 0.3195491824299097 } ] }, "comparison": { "delta_mean": 0.07120507396757603, "idea_wins": 0, "n_pairs": 8, "per_seed_diffs": [ 0.05258098244667053, 0.06303387880325317, 0.10102905333042145, 0.024073749780654907, 0.12098173797130585, 0.07209694385528564, 0.0679224282503128, 0.06792181730270386 ], "p_value": 0.0081, "mde": 0.02453005366966053, "mde_rel_pct": 14.730432336430782, "verdict": "idea worse (significant)", "system_worked": false }, "selection": { "baseline_best_cfg": { "lr": 0.001, "epochs": 12 }, "idea_best_cfg": { "lr": 0.001, "epochs": 12 }, "shared_lr_union": [ 0.001, 0.003, 0.006 ] }, "structural_match": "multi-token correlated sequence window", "mechanism_signature": { "quantity": "trained-model pair inclusion mass", "predicted_direction": "compatibility boost increases joint inclusion", "boost": 2.0, "observed_before": 0.039688840508461, "observed_after": 0.22817423939704895, "observed_change": 0.18848538398742676, "confirmed": true }, "protocol_note": "Eight paired seeds; three shared learning rates; reduced 1200/400 samples and 12 epochs for runtime." }, "how_to_run": "/home/maxwelhelp/main/bin/python3 bench_collision_subset.py", "files": [ "bench_collision_subset.py", "bench_report.json" ], "limitations": "The benchmark used the standard built-in sequence forecast track, reduced to 1,200 training and 400 test examples with 12 epochs for runtime. It did not test vision, dynamics, longer neighborhoods, larger subset sizes, equalized parameter/FLOP counts, or the original synthetic collision-label task in this round; the exact subset router also has higher local computational and parameter overhead than ordinary multi-head attention.", "system_verdict": "partial", "practical_verdict": "harms", "mechanism_ok": 1, "system_judged": true }