Стенд-проверка (stage-2) · промт оператора:

(универсальный)

Ответ агента:

{ "worked": true, "confidence": 8, "verdict": "Implemented a Fisher–KPP-gated transformer_tiny on the sequence forecasting benchmark, using the same base architecture, optimizer, epochs, and paired datasets as baseline. Across 8 paired seeds, idea test MSE was 0.38069 versus baseline 0.44760, delta=-0.06691, with permutation p=0.0081 and all 8 paired wins, so the benchmark verdict is idea better (significant). The independent trained-model mechanism signature was not confirmed: gate-vs-one-step-PDE RMSE was 0.4042 against a 0.15 tolerance; therefore the task win is real empirically, but attribution to quantitatively faithful Fisher–KPP dynamics is not established.", "metrics": { "baseline": "sequence/transformer_tiny, tuned best lr=0.0015 epochs=12; 8-seed test MSE mean 0.4476016909, per-seed [0.39361495,0.38908216,0.40740371,0.50282270,0.43237618,0.48851299,0.52378368,0.44321716]", "idea": "Fisher-KPP gated transformer_tiny, best lr=0.0015 epochs=12; 8-seed test MSE mean 0.3806905746, per-seed [0.36546338,0.33078253,0.37874427,0.44937277,0.35211891,0.41673592,0.44355533,0.30875149]", "delta_mean": "-0.0669111162 (idea lower/better), all 8 paired wins, permutation p=0.0081", "baseline_sweep": "lr 0.0015: 0.4232308790; lr 0.003: 0.4927415997; lr 0.006: 0.7217541784 on 4 tuning seeds", "mechanism_signature": "Predicted-vs-observed trained gate RMSE 0.4042392542, tolerance 0.15, confirmed=false; observed mean half-gate front index 0.37890625", "math_sanity": "Deterministic Fisher-KPP front relative errors 5.13%, 5.31%, and 10.28% for D=0.1,0.25,0.5; stability slope absolute errors 0.00635, approximately 0, and 0.00615 for r=-0.5,0,0.5." }, "bench_report": { "bench_version": 1, "track": "sequence", "model": "transformer_tiny", "metric_direction": "lower is better", "n_seeds": 8, "baseline": { "best_cfg": { "lr": 0.0015, "epochs": 12 }, "sweep": [ { "cfg": { "lr": 0.0015, "epochs": 12 }, "mean": 0.42323087900877 }, { "cfg": { "lr": 0.003, "epochs": 12 }, "mean": 0.4927415996789932 }, { "cfg": { "lr": 0.006, "epochs": 12 }, "mean": 0.721754178404808 } ], "full": { "mean": 0.44760169088840485, "std": 0.0483900971330472, "per_seed": [ 0.3936149477958679, 0.3890821635723114, 0.40740370750427246, 0.5028226971626282, 0.4323761761188507, 0.4885129928588867, 0.5237836837768555, 0.4432171583175659 ], "n": 8 } }, "idea": { "config": { "lr": 0.0015, "epochs": 12 }, "per_seed": [ 0.36546337604522705, 0.33078253269195557, 0.3787442743778229, 0.4493727684020996, 0.35211890935897827, 0.4167359173297882, 0.4435553252696991, 0.30875149369239807 ], "mean": 0.3806905746459961 }, "comparison": { "delta_mean": -0.06691111624240875, "idea_wins": 8, "n_pairs": 8, "per_seed_diffs": [ -0.02815157175064087, -0.058299630880355835, -0.028659433126449585, -0.053449928760528564, -0.08025726675987244, -0.07177707552909851, -0.08022835850715637, -0.13446566462516785 ], "p_value": 0.0081, "mde": 0.028554757083691817, "mde_rel_pct": 6.379501611581497, "verdict": "idea better (significant)", "system_worked": true }, "mechanism_signature": { "quantity": "trained gate vs one-step Fisher-KPP prediction", "predicted_gate_rmse": 0.4042392542224135, "observed_front_index_mean": 0.37890625, "tolerance": 0.15, "confirmed": false }, "math_sanity": { "speed_scaling": [ { "D": 0.1, "r": 1.0, "predicted": 0.6324555320336759, "observed": 0.6649048996195985, "relative_error": 0.05130695510177757 }, { "D": 0.25, "r": 1.0, "predicted": 1.0, "observed": 0.9468693074772295, "relative_error": 0.05313069252277047 }, { "D": 0.5, "r": 1.0, "predicted": 1.4142135623730951, "observed": 1.268869715478862, "relative_error": 0.1027736197426516 } ], "stability_boundary": [ { "r": -0.5, "predicted": -0.5, "observed": -0.5063520554424232, "abs_error": 0.006352055442423232 }, { "r": 0.0, "predicted": 0.0, "observed": -1.083053609166525e-14, "abs_error": 1.083053609166525e-14 }, { "r": 0.5, "predicted": 0.5, "observed": 0.49384613004092737, "abs_error": 0.006153869959072633 } ] }, "protocol_note": "8 paired seeds; baseline sweep and idea sweep use identical lr/epoch union; n_train=400." }, "how_to_run": "/home/maxwelhelp/main/bin/python3 bench_experiment.py", "files": [ "bench_experiment.py", "bench_report.json", "bench_stdout.txt" ], "limitations": "Only the sequence track was tested; no vision, tabular, or dynamics transfer was evaluated. The benchmark used 400 training and 200 test examples, 12 epochs, and deterministic noise-disabled gating for the reported best configuration. The mechanism signature used a one-step predicted gate comparison rather than a long-depth impulse-front measurement, and failed its preset tolerance; no claim is made that the task improvement is caused specifically by faithful stochastic Fisher–KPP behavior.", "system_verdict": "worked", "practical_verdict": "helps", "mechanism_ok": 0, "system_judged": true }