Стенд-проверка (stage-2) · промт оператора:

(универсальный)

Ответ агента:

{ "worked": false, "confidence": 10, "verdict": "The Hankel Residual Observer was implemented and evaluated against independently trained transformer_tiny systems on the structurally matched registered sequence track. It significantly worsened test MSE: baseline 0.5443 versus idea 0.7173, paired delta +0.1730 with permutation p=0.0081. The trained-model mechanism signature was not confirmed because residual forecast MSE exceeded the naive residual MSE.", "metrics": { "baseline": "Best lr=0.001; mean test MSE 0.5443045375, std 0.0343916344.", "idea": "Best lr=0.001; mean test MSE 0.7173206874, std 0.0432494822; paired delta +0.1730161499, p=0.0081, 0/8 wins." }, "bench_report": { "bench_version": 1, "track": "sequence", "model": "transformer_tiny", "metric_direction": "lower is better", "n_seeds": 8, "baseline": { "best_cfg": { "lr": 0.001 }, "sweep": [ { "cfg": { "lr": 0.001 }, "mean": 0.5443045374850393 }, { "cfg": { "lr": 0.003 }, "mean": 0.6654394097127154 }, { "cfg": { "lr": 0.006 }, "mean": 0.8961613556937273 } ], "full": { "mean": 0.5443045374850393, "std": 0.034391634415997095, "per_seed": [ 0.563120002561451, 0.4667357694115086, 0.5482129393734809, 0.5391273206172196, 0.5435938104924675, 0.542868393619944, 0.5989920214133407, 0.5517860423909019 ], "n": 8 } }, "idea": { "mean": 0.7173206874260685, "std": 0.04324948219202462, "per_seed": [ 0.7285622398292526, 0.6195935527509657, 0.7682973969911581, 0.7145800123187942, 0.7425737796077112, 0.6909476920862011, 0.7539289132268427, 0.7200819125976213 ], "n": 8 }, "comparison": { "delta_mean": 0.1730161499410291, "idea_wins": 0, "n_pairs": 8, "per_seed_diffs": [ 0.1654422372678016, 0.1528577833394571, 0.22008445761767714, 0.1754526917015742, 0.19897996911524374, 0.14807929846625711, 0.154936891813502, 0.16829587020671943 ], "p_value": 0.0081, "mde": 0.020796525052684568, "mde_rel_pct": 3.8207517337215253, "verdict": "idea worse (significant)", "system_worked": false }, "mechanism_signature": { "residual_forecast_mse": 0.7368731066703664, "residual_naive_mse": 0.545037575640239, "correction_mse": 0.7173206874260685, "base_mse": 0.5443045374850393, "mean_gamma": 0.9632895999502775, "active_fraction": 0.9775, "mean_mismatch_q": 0.01453749365700519, "confirmed": false }, "idea_sweep": [ { "cfg": { "lr": 0.001 }, "mean": 0.7173206874260685, "std": 0.04324948219202462 }, { "cfg": { "lr": 0.003 }, "mean": 1.0912813241829704, "std": 0.5463389443333877 }, { "cfg": { "lr": 0.006 }, "mean": 1.320354687932311, "std": 0.6682789006276265 } ], "protocol": { "seeds": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "epochs": 8, "n_train": 400, "n_test": 400, "shared_lr_grid": [ { "lr": 0.001 }, { "lr": 0.003 }, { "lr": 0.006 } ], "observer": { "t_ini": 8, "horizon": 1, "max_cols": 40, "ridge": 0.01, "tau": 1.0 } } }, "how_to_run": "/home/maxwelhelp/main/bin/python3 bench_hankel.py", "files": [ "bench_hankel.py", "bench_report.json" ], "limitations": "Only the registered built-in sequence track was tested. No external real-world dataset, dynamics track, observer latency, or memory benchmark was included.", "system_verdict": "failed", "practical_verdict": "harms", "mechanism_ok": 0, "system_judged": true }