Стенд-проверка (stage-2) · промт оператора:

(универсальный)

Ответ агента:

{ "worked": false, "confidence": 10, "verdict": "Built and evaluated the Kernel-Prompted Random Transformer on the structurally matched sequence forecast track using the same frozen random-attention architecture for both systems. The mechanism signature was confirmed quantitatively: prompt-norm ratio 4.0001 versus the predicted 4.0 and best affine-logit residual 8.8e-14. However, the idea had test MSE 1.7407 versus baseline 0.9357, paired delta +0.8050 with permutation p=0.0081, so it is significantly worse and does not count as a win.", "metrics": { "baseline": "Best lr=0.01 from shared grid [0.001, 0.003, 0.01]; full 8-seed MSE mean 0.9356768578, std 0.1068674329.", "idea": "Best tested setting sigma=0.8, lr=0.01; full 8-seed MSE mean 1.7406605533, std 0.1589719376. Paired delta idea-baseline=+0.8049836955, p=0.0081, idea wins 0/8.", "mechanism_signature": "Prompt norms sigma=[0.4,0.6,0.8] were [2036.9190,905.2893,509.2197], observed 0.4/0.8 ratio 4.000079 versus predicted 4.0; observed best logit residual 8.765e-14; confirmed=true." }, "how_to_run": "/home/maxwelhelp/main/bin/python3 stage2_bench.py", "files": [ "stage2_bench.py", "bench_report.json" ], "limitations": "Only the sequence track was tested; CIFAR/MNIST transfer, other built-in tracks, full multi-layer/multi-head transformer comparison, inference latency/FLOP measurements, and gradient-optimized prompts beyond the matched frozen-attention prompt baseline were not tested.", "bench_report": { "bench_version": 1, "track": "sequence", "model": "frozen_random_attention_prompt", "metric_direction": "lower is better", "n_seeds": 8, "baseline": { "best_cfg": { "lr": 0.01 }, "sweep": [ { "cfg": { "lr": 0.001 }, "mean": 1.0786777288 }, { "cfg": { "lr": 0.003 }, "mean": 1.0726252794 }, { "cfg": { "lr": 0.01 }, "mean": 0.921186164 } ], "full": { "mean": 0.9356768578, "std": 0.1068674329, "per_seed": [ 1.0163524151, 0.7307133079, 1.0717743635, 0.8659045696, 0.8926808238, 1.0664112568, 0.9147705436, 0.9268075824 ], "n": 8 } }, "idea": { "best_cfg": { "sigma": 0.8, "lr": 0.01 }, "mean": 1.7406605533, "std": 0.1589719376, "per_seed": [ 2.092819767, 1.6468335788, 1.6924736705, 1.6685188268, 1.6360904797, 1.7143411545, 1.8954980475, 1.5787089015 ], "n": 8 }, "comparison": { "delta_mean": 0.8049836955, "idea_wins": 0, "n_pairs": 8, "p_value": 0.0081, "verdict": "idea worse (significant)", "system_worked": false }, "mechanism_signature": { "track_choice": "sequence: multi-token temporal-window correlations require sequence attention", "prediction": "prompt norm scales as sigma^-2; affine logit residual is near zero", "observed_prompt_norms": { "0.4": 2036.9190406, "0.6": 905.289309, "0.8": 509.2196552 }, "predicted_norm_ratio_sigma_0.4_to_0.8": 4.0, "observed_norm_ratio_sigma_0.4_to_0.8": 4.000079376, "observed_best_logit_residual": 8.765288996e-14, "confirmed": true } }, "system_verdict": "partial", "practical_verdict": "harms", "mechanism_ok": 1, "system_judged": true }