# Стенд-проверка (stage-2) · промт оператора:

(универсальный)

Ответ агента:

{
  "worked": false,
  "confidence": 9,
  "verdict": "The registered custom router_regime_regression benchmark compared matched end-to-end six-expert MoE systems: standard softmax routing with Adam against fixed-m external-field routing with exact covariance pseudoinverse router gradients and resistance trust scaling. The trained-model mechanism signature was confirmed and the math sanity checks passed. Nevertheless, the best idea result was significantly worse than the tuned baseline on test MSE, so this idea does not provide a benchmark win.",
  "metrics": {
    "baseline": "Mean test MSE 0.3102635518 over 8 seeds, std 0.0749747019; best lr=0.012, temp=0.7.",
    "idea": "Best mean test MSE 0.3417467810 over 8 seeds, std 0.0732672876; lr=0.012, temp=0.7, rho=0.3. Paired delta +0.0314832292, p=0.0081, 0/8 wins.",
    "mechanism_signature": "confirmed=true; mean observed cap=0.2041564326 and mean predicted cap=0.2041564326.",
    "math_sanity": "max resistance ratio=0.9998409991; minimum covariance-bound eigenvalue=-1.186e-16; trust-cap maximum absolute error=2.776e-17."
  },
  "bench_report": {
    "bench_version": 1,
    "track": "router_regime_regression",
    "model": "custom_mlp_moe",
    "metric_direction": "lower is better",
    "n_seeds": 8,
    "baseline": {
      "best_cfg": {
        "lr": 0.012,
        "temp": 0.7
      },
      "sweep": [
        {
          "cfg": {
            "lr": 0.003,
            "temp": 0.7
          },
          "mean": 0.5712137892842293
        },
        {
          "cfg": {
            "lr": 0.003,
            "temp": 1.0
          },
          "mean": 0.6234312355518341
        },
        {
          "cfg": {
            "lr": 0.006,
            "temp": 0.7
          },
          "mean": 0.4075416550040245
        },
        {
          "cfg": {
            "lr": 0.006,
            "temp": 1.0
          },
          "mean": 0.44819550961256027
        },
        {
          "cfg": {
            "lr": 0.012,
            "temp": 0.7
          },
          "mean": 0.27756691724061966
        },
        {
          "cfg": {
            "lr": 0.012,
            "temp": 1.0
          },
          "mean": 0.31893689185380936
        }
      ],
      "full": {
        "mean": 0.3102635517716408,
        "std": 0.07497470190378723,
        "per_seed": [
          0.30197641253471375,
          0.24877610802650452,
          0.29255229234695435,
          0.26696285605430603,
          0.28872963786125183,
          0.5001022219657898,
          0.26213279366493225,
          0.3208760917186737
        ],
        "n": 8
      }
    },
    "idea": {
      "cfg": {
        "lr": 0.012,
        "temp": 0.7,
        "rho": 0.3
      },
      "per_seed": [
        0.3346004784107208,
        0.27418217062950134,
        0.3511717915534973,
        0.32937872409820557,
        0.3155486285686493,
        0.5077916979789734,
        0.2751004695892334,
        0.34620028734207153
      ],
      "mean": 0.3417467810213566,
      "std": 0.07326728763277315,
      "signature_summary": {
        "mean_observed_cap": 0.2041564326380133,
        "mean_predicted_cap": 0.2041564326380133
      }
    },
    "comparison": {
      "delta_mean": 0.031483229249715805,
      "idea_wins": 0,
      "n_pairs": 8,
      "per_seed_diffs": [
        0.03262406587600708,
        0.025406062602996826,
        0.05861949920654297,
        0.062415868043899536,
        0.02681899070739746,
        0.007689476013183594,
        0.012967675924301147,
        0.025324195623397827
      ],
      "p_value": 0.0081,
      "mde": 0.01642202588917077,
      "mde_rel_pct": 5.292927833578616,
      "verdict": "idea worse (significant)",
      "system_worked": false
    },
    "mechanism_signature": {
      "prediction": "Resistance-scaled natural-gradient router updates obey the pairwise cap on trained neural-network updates.",
      "predicted_vs_observed": {
        "mean_observed_cap": 0.2041564326380133,
        "mean_predicted_cap": 0.2041564326380133
      },
      "confirmed": true,
      "math_sanity": {
        "max_resistance_ratio": 0.9998409991250671,
        "min_covariance_bound_eigenvalue": -1.1862584540283742e-16,
        "trust_cap_max_abs_error": 2.7755575615628914e-17,
        "passed": true
      }
    },
    "custom_track": {
      "name": "router_regime_regression",
      "file": "custom_router_track.py",
      "domain": "moe-routing"
    },
    "idea_sweep": [
      {
        "cfg": {
          "lr": 0.012,
          "temp": 0.7,
          "rho": 0.15
        },
        "mean": 0.3429313190281391
      },
      {
        "cfg": {
          "lr": 0.012,
          "temp": 0.7,
          "rho": 0.3
        },
        "mean": 0.3417467810213566
      },
      {
        "cfg": {
          "lr": 0.012,
          "temp": 0.7,
          "rho": 0.6
        },
        "mean": 0.34471020475029945
      },
      {
        "cfg": {
          "lr": 0.003,
          "temp": 0.7,
          "rho": 0.3
        },
        "mean": 0.6366909965872765
      },
      {
        "cfg": {
          "lr": 0.006,
          "temp": 0.7,
          "rho": 0.3
        },
        "mean": 0.47830405086278915
      }
    ]
  },
  "how_to_run": "/home/maxwelhelp/main/bin/python3 run_bench.py",
  "files": [
    "custom_router_track.py",
    "run_bench.py",
    "bench_report.json"
  ],
  "limitations": "Only the registered custom 400-sample router-regression track was tested. Built-in CIFAR, language-modeling, sequence, and dynamics tracks were not structurally matched to expert routing and were not used. Large-scale MoE, communication cost, expert-load CV, dead-expert fraction, and gradient-spike analysis were not tested. The benchmark used six small experts and exact subset enumeration, so scalability was not evaluated.",
  "system_verdict": "partial",
  "practical_verdict": "harms",
  "mechanism_ok": 1,
  "system_judged": true
}