Стенд-проверка (stage-2) · промт оператора:

(универсальный)

Ответ агента:

{ "worked": false, "confidence": 9, "verdict": "KL Mirror-Prox was implemented as a two-stage KL prior update in a matched three-expert MoE system on the registered correlated_token_moe_regression track. It produced a large reduction in observed expert-load variance, but the primary test MSE improvement was not statistically significant, so the bench verdict is no measurable effect and the idea did not win under the required criterion.", "metrics": { "baseline": "8-seed test MSE mean 0.04093059757724404; tuned baseline sweep best cfg {lr:0.006, eta:0.03}.", "idea": "8-seed test MSE mean approximately 0.04056028323248028; paired delta -0.0003703143447637558; 5/8 wins; permutation p=0.20205. Routing-load variance was substantially lower for the idea." }, "bench_report": { "bench_version": 1, "track": "correlated_token_moe_regression", "model": "custom_moe_shared", "metric_direction": "lower is better", "n_seeds": 8, "baseline": { "best_cfg": { "lr": 0.006, "eta": 0.03 }, "sweep": [ { "cfg": { "lr": 0.001, "eta": 0.03 }, "mean": 0.06951743829995394 }, { "cfg": { "lr": 0.001, "eta": 0.1 }, "mean": 0.06951743829995394 }, { "cfg": { "lr": 0.001, "eta": 0.3 }, "mean": 0.06951743829995394 }, { "cfg": { "lr": 0.003, "eta": 0.03 }, "mean": 0.043228451162576675 }, { "cfg": { "lr": 0.003, "eta": 0.1 }, "mean": 0.043228451162576675 }, { "cfg": { "lr": 0.003, "eta": 0.3 }, "mean": 0.043228451162576675 }, { "cfg": { "lr": 0.006, "eta": 0.03 }, "mean": 0.04188114870339632 }, { "cfg": { "lr": 0.006, "eta": 0.1 }, "mean": 0.04188114870339632 }, { "cfg": { "lr": 0.006, "eta": 0.3 }, "mean": 0.04188114870339632 } ], "full": { "mean": 0.04093059757724404, "std": 0.0038662018493219787, "per_seed": [ 0.036128461360931396, 0.04280463978648186, 0.04104797914624214, 0.047543514519929886, 0.038935478776693344, 0.04488188028335571, 0.04060979187488556, 0.03549303486943245 ], "n": 8 } }, "idea": { "mean": 0.04056028323248028, "std": 0.003431, "per_seed": [ 0.03617550805211067, 0.042327430099248886, 0.040069837123155594, 0.04594426974654198, 0.039504799991846085, 0.04505091905593872, 0.04003051668405533, 0.03537898510694504 ], "n": 8 }, "comparison": { "delta_mean": -0.0003703143447637558, "idea_wins": 5, "n_pairs": 8, "per_seed_diffs": [ 4.704669117927551e-05, -0.0004772096872329712, -0.0009781420230865479, -0.001599244773387909, 0.0005693212151527405, 0.0001690387725830078, -0.0005792751908302307, -0.0001140497624874115 ], "p_value": 0.20205, "mde": 0.0005783116111008068, "mde_rel_pct": 1.412907813059459, "verdict": "no measurable effect", "system_worked": false }, "mechanism_signature": { "predicted_prior_vs_observed_load_mae": 0.114, "predicted_prior_mean": [ 0.4054288476705551, 0.27825858145952226, 0.31631257086992265 ], "observed_load_mean": [ 0.3356913700699806, 0.3363957405090332, 0.327912874519825 ], "confirmed": false, "note": "Measured on trained MoE systems; the two-stage update stabilized realized routing loads, but the predicted prior did not quantitatively track final observed load closely enough under the stated tolerance." } }, "how_to_run": "/home/maxwelhelp/main/bin/python3 run_bench.py", "files": [ "run_bench.py", "bench_report.json" ], "limitations": "The benchmark used a small registered correlated-token MoE regression track with 400 training and 200 test examples, three experts, 18 epochs, and a custom optimizer loop because the intervention changes the training update. It did not test larger MoE systems, top-1 dispatch capacity overflow, CIFAR-scale routing, longer training, or wall-clock overhead. The baseline eta field is inert for the one-stage update but was included in the shared search-space parity grid.", "system_verdict": "failed", "practical_verdict": "no_effect", "mechanism_ok": 0, "system_judged": true }