Стенд-проверка (stage-2) · промт оператора:

(универсальный)

Ответ агента:

{ "worked": false, "confidence": 9, "verdict": "The KL-TopK PCA bottleneck was tested end-to-end in the registered sequence/transformer_tiny benchmark against the same transformer with coordinate top-k. PCA improved the standalone residual-energy sanity check, but task MSE was slightly worse and the paired permutation test was non-significant, so the idea did not produce the required benchmark win.", "bench_report": { "bench_version": 1, "track": "sequence", "model": "transformer_tiny", "metric_direction": "lower is better", "n_seeds": 8, "baseline": { "best_cfg": { "lr": 0.004, "keep": 12 }, "sweep": [ { "cfg": { "lr": 0.002, "keep": 4 }, "mean": 0.7055840939283371 }, { "cfg": { "lr": 0.002, "keep": 8 }, "mean": 0.6527759283781052 }, { "cfg": { "lr": 0.002, "keep": 12 }, "mean": 0.6173136383295059 }, { "cfg": { "lr": 0.003, "keep": 4 }, "mean": 0.6469062119722366 }, { "cfg": { "lr": 0.003, "keep": 8 }, "mean": 0.6016789823770523 }, { "cfg": { "lr": 0.003, "keep": 12 }, "mean": 0.5432282239198685 }, { "cfg": { "lr": 0.004, "keep": 4 }, "mean": 0.5868831127882004 }, { "cfg": { "lr": 0.004, "keep": 8 }, "mean": 0.5453523471951485 }, { "cfg": { "lr": 0.004, "keep": 12 }, "mean": 0.5168638676404953 } ], "full": { "mean": 0.5058855153620243, "std": 0.03455579185574975, "per_seed": [ 0.5703754425048828, 0.48083919286727905, 0.5038946270942688, 0.5123462080955505, 0.5410808324813843, 0.5025615096092224, 0.48530539870262146, 0.4506809115409851 ], "n": 8 } }, "idea": { "mean": 0.5126509107649326, "std": 0.03918740779455468, "per_seed": [ 0.5694426894187927, 0.48401522636413574, 0.5518521070480347, 0.500401496887207, 0.5515084862709045, 0.5079619288444519, 0.4907732903957367, 0.44525206089019775 ], "n": 8 }, "comparison": { "delta_mean": 0.006765395402908325, "idea_wins": 3, "n_pairs": 8, "per_seed_diffs": [ -0.0009327530860900879, 0.0031760334968566895, 0.04795747995376587, -0.011944711208343506, 0.010427653789520264, 0.005400419235229492, 0.005467891693115234, -0.0054288506507873535 ], "p_value": 0.3883, "mde": 0.015105943689001283, "mde_rel_pct": 2.9860399695751503, "verdict": "no significant win", "system_worked": false }, "mechanism_signature": { "math_check": { "pca_residual": 1.910288129226306, "rotation_residual": 2.8467599006415716, "pca_le_rotation": true, "topk_formula_error": 0.0 }, "trained_model_signature": { "predicted_pca_retained_energy": 5.444458019758442, "observed_topk_retained_fraction": 0.9996704459190369, "mean_selection_gap": 0.10093563050031662, "confirmed": true, "source": "trained sequence model hidden states" }, "idea_cfg": { "lr": 0.004, "keep": 12 }, "track_match": "multi-token sequence forecast; activation bottleneck inserted in the transformer hidden sequence." }, "idea_sweep": [ { "cfg": { "lr": 0.004, "keep": 12 }, "result": { "mean": 0.5126509107649326, "std": 0.03918740779455468, "per_seed": [ 0.5694426894187927, 0.48401522636413574, 0.5518521070480347, 0.500401496887207, 0.5515084862709045, 0.5079619288444519, 0.4907732903957367, 0.44525206089019775 ], "n": 8 } }, { "cfg": { "lr": 0.002, "keep": 12 }, "result": { "mean": 0.5912451855838299, "std": 0.05238796828411388, "per_seed": [ 0.6840536594390869, 0.5757109522819519, 0.5977234843063354, 0.5731315016746521, 0.6317427158355713, 0.6175632476806641, 0.553534746170044, 0.4965011775493622 ], "n": 8 } }, { "cfg": { "lr": 0.004, "keep": 12 }, "result": { "mean": 0.5126509107649326, "std": 0.03918740779455468, "per_seed": [ 0.5694426894187927, 0.48401522636413574, 0.5518521070480347, 0.500401496887207, 0.5515084862709045, 0.5079619288444519, 0.4907732903957367, 0.44525206089019775 ], "n": 8 } } ] }, "metrics": { "baseline": "Full paired baseline test MSE mean 0.5058855 (lr=0.004, keep=12).", "idea": "Full paired PCA top-k test MSE mean 0.5126509 (lr=0.004, keep=12); delta +0.0067654, p=0.3883.", "math": "PCA residual 1.910288 versus random-rotation residual 2.846760; formula implementation error 0.0." }, "how_to_run": "/home/maxwelhelp/main/bin/python3 stage2_kl_topk_bench.py", "files": [ "stage2_kl_topk_bench.py", "bench_report.json" ], "limitations": "Only the registered sequence track and tiny transformer were tested. Learned rotations, covariance refresh, larger models, explicit activation-memory/index accounting, and wall-clock compression overhead were not tested.", "system_verdict": "failed", "practical_verdict": "no_effect", "mechanism_ok": 0, "system_judged": true }