Стенд-проверка (stage-2) · промт оператора:

(универсальный)

Ответ агента:

{ "worked": false, "confidence": 9, "verdict": "The registered custom track expert_balanced_regression was accepted by the bench smoke test and evaluated with identical end-to-end top-1 MoE systems, paired seeds, baseline sweep, and make_report(). VDC improved mean test MSE from 0.2129017226 to 0.2056338880, but the paired permutation test was not significant (p=0.08055), so the required benchmark verdict is no significant win. The trained-model mechanism signature was confirmed: observed maximum leaf prefix discrepancy was 0.875 versus the prediction <=1.0, with final load variance 0.0.", "metrics": { "baseline": "Registered random-routing top-1 MoE, tuned lr=0.01, epochs=12: mean test MSE 0.2129017226; per-seed [0.2303046435, 0.1982028186, 0.1861430258, 0.2568784952, 0.2142492831, 0.1867903024, 0.2180261165, 0.2126190960].", "idea": "Registered VDC-routing top-1 MoE, lr=0.01, rho=1.0, epochs=12: mean test MSE 0.2056338880; paired delta -0.0072678346; 6/8 wins; p=0.08055; observed prefix discrepancy 0.875 and final load variance 0.0." }, "bench_report": { "bench_version": 1, "track": "expert_balanced_regression", "model": "top1_moe_shared", "metric_direction": "lower is better", "n_seeds": 8, "baseline": { "best_cfg": { "lr": 0.01, "epochs": 12, "rho": 0.5 }, "sweep": [ { "cfg": { "lr": 0.001, "epochs": 12, "rho": 0.5 }, "mean": 0.5207640379667282 }, { "cfg": { "lr": 0.001, "epochs": 12, "rho": 1.0 }, "mean": 0.5207640379667282 }, { "cfg": { "lr": 0.003, "epochs": 12, "rho": 0.5 }, "mean": 0.3618839532136917 }, { "cfg": { "lr": 0.003, "epochs": 12, "rho": 1.0 }, "mean": 0.3618839532136917 }, { "cfg": { "lr": 0.01, "epochs": 12, "rho": 0.5 }, "mean": 0.21788224577903748 }, { "cfg": { "lr": 0.01, "epochs": 12, "rho": 1.0 }, "mean": 0.21788224577903748 } ], "full": { "mean": 0.21290172263979912, "std": 0.02201949504273137, "per_seed": [ 0.23030464351177216, 0.19820281863212585, 0.18614302575588226, 0.25687849521636963, 0.21424928307533264, 0.18679030239582062, 0.21802611649036407, 0.2126190960407257 ], "n": 8 } }, "idea": { "mean": 0.2056338880211115, "std": 0.020574444566337256, "per_seed": [ 0.2081810086965561, 0.20322394371032715, 0.18413016200065613, 0.24816179275512695, 0.19936268031597137, 0.17569208145141602, 0.20796272158622742, 0.21835671365261078 ], "n": 8, "config": { "lr": 0.01, "epochs": 12, "rho": 1.0 } }, "comparison": { "delta_mean": -0.00726783461868763, "idea_wins": 6, "n_pairs": 8, "per_seed_diffs": [ -0.022123634815216064, 0.005021125078201294, -0.0020128637552261353, -0.008716702461242676, -0.014886602759361267, -0.011098220944404602, -0.010063394904136658, 0.005737617611885071 ], "p_value": 0.08055, "mde": 0.008064204749019211, "mde_rel_pct": 3.7877592764539316, "verdict": "no significant win", "system_worked": false }, "mechanism_signature": { "prediction": "trained VDC assignments maintain max leaf prefix discrepancy <=1", "predicted_max_discrepancy": 1.0, "observed_mean_max_discrepancy": 0.875, "observed_final_load_variance": 0.0, "confirmed": true }, "custom_track": { "name": "expert_balanced_regression", "file": "expert_balanced_regression.py", "domain": "moe-routing" } }, "how_to_run": "/home/maxwelhelp/main/bin/python3 registered_bench.py", "files": [ "vdc_bench.py", "registered_bench.py", "bench_report.json", "/home/maxwelhelp/all/math2nn/bench/custom_tracks/expert_balanced_regression.py" ], "limitations": "The custom task is a small synthetic regression rather than a standard Switch Transformer or large-scale language/vision benchmark. Capacity overflow/drop-rate behavior was not implemented in the neural training loop, nested-group discrepancy beyond leaf prefixes was not measured, and the baseline routing sweep used random routing rather than a learned greedy router.", "system_verdict": "partial", "practical_verdict": "inconclusive", "mechanism_ok": 1, "system_judged": true }