Стенд-проверка (stage-2) · промт оператора:

(универсальный)

Ответ агента:

{ "worked": false, "confidence": 9, "verdict": "Implemented Fixed-Projection Temporal Plasticity as a local training intervention on an identical sigmoid-input transformer and evaluated it on the structurally matched sequence forecast track. The mechanism signature was confirmed quantitatively at NN scale: positive trained-model projection alignment (0.02075) and zero saturated activity. However, the idea had higher MSE than the tuned baseline (1.00834 vs 1.00688), with paired delta +0.001465 and permutation p=0.01485, yielding a significant loss; therefore worked=false.", "metrics": { "baseline": "Adam sigmoid-input transformer, best lr=0.001; full 8-seed test MSE 1.006878 ± 0.171372; per-seed [0.991178, 0.755488, 1.263213, 0.788392, 1.153331, 0.941530, 1.184309, 0.977584]. Baseline sweep means: lr 0.001=0.949568, 0.003=1.508308, 0.01=3.829346.", "idea": "Fixed-projection temporal local update, best lr=0.001; full 8-seed test MSE 1.008343 ± 0.172025; per-seed [0.990945, 0.756263, 1.265931, 0.789582, 1.153704, 0.943642, 1.188511, 0.978167]. Idea sweep means: lr 0.001=1.008343, 0.003=1.482208, 0.01=4.900032.", "paired_comparison": "delta_mean=+0.001464881; idea_wins=1/8; permutation p_value=0.01485; verdict='idea worse (significant)'.", "mechanism_signature": "mean hidden activity=0.494660; saturated fraction=0.0; observed projection-alignment statistic=0.020752; predicted positive alignment=True; confirmed=True." }, "bench_report": { "bench_version": 1, "track": "sequence", "model": "transformer_tiny", "metric_direction": "lower is better", "n_seeds": 8, "baseline": { "best_cfg": { "lr": 0.001 }, "sweep": [ { "cfg": { "lr": 0.001 }, "mean": 0.9495677947998047 }, { "cfg": { "lr": 0.003 }, "mean": 1.5083084404468536 }, { "cfg": { "lr": 0.01 }, "mean": 3.8293468952178955 } ], "full": { "mean": 1.006878174841404, "std": 0.1713716806448823, "per_seed": [ 0.9911782145500183, 0.7554877996444702, 1.2632131576538086, 0.7883920073509216, 1.1533313989639282, 0.9415301084518433, 1.1843085289001465, 0.977584183216095 ], "n": 8 }, "idea_grid": [ { "cfg": { "lr": 0.001 }, "mean": 1.008343055844307 }, { "cfg": { "lr": 0.003 }, "mean": 1.4822083115577698 }, { "cfg": { "lr": 0.01 }, "mean": 4.900031566619873 } ] }, "idea": { "mean": 1.008343055844307, "std": 0.1720253375372032, "per_seed": [ 0.9909452795982361, 0.7562634944915771, 1.2659306526184082, 0.7895816564559937, 1.153704047203064, 0.9436419606208801, 1.1885106563568115, 0.9781666994094849 ], "n": 8 }, "comparison": { "delta_mean": 0.0014648810029029846, "idea_wins": 1, "n_pairs": 8, "per_seed_diffs": [ -0.00023293495178222656, 0.0007756948471069336, 0.0027174949645996094, 0.0011896491050720215, 0.0003726482391357422, 0.0021118521690368652, 0.004202127456665039, 0.0005825161933898926 ], "p_value": 0.01485, "mde": 0.0012193019578819895, "mde_rel_pct": 0.12109726760876957, "verdict": "idea worse (significant)", "system_worked": false }, "mechanism_signature": { "mean_hidden": 0.49465957656502724, "saturated_fraction": 0.0, "projection_alignment": 0.020752153369809595, "predicted_projection_alignment_positive": true, "confirmed": true }, "protocol_notes": { "architecture_match": "identical sigmoid-input transformer; only inp training rule differs", "track_justification": "sequence forecast contains ordered multi-token temporal windows", "budget": { "n_train": 400, "n_test": 400, "epochs": 5, "batch": 128, "lr_union": [ 0.001, 0.003, 0.01 ] }, "custom_track": null } }, "how_to_run": "/home/maxwelhelp/main/bin/python3 stage2_bench.py", "files": [ "stage2_bench.py", "bench_report.json", "bench_stdout.txt" ], "limitations": "Used the benchmark's reduced 400-train/400-test budget and 5 epochs rather than the README's larger default sizes; did not test vision, tabular, or dynamics tracks. The intervention updates only the sigmoid input projection while the transformer body/head remain Adam-trained, which is the chosen operationalization of replacing encoder backpropagation.", "system_verdict": "partial", "practical_verdict": "harms", "mechanism_ok": 1, "system_judged": true }