# Стенд-проверка (stage-2) · промт оператора:

(универсальный)

Ответ агента:

{
  "worked": true,
  "confidence": 8,
  "verdict": "Implemented a matched end-to-end permutation-invariant set model and compared it with a label-sensitive flattened MLP on the structurally appropriate correlated-token regression benchmark. Across 8 paired seeds, the idea achieved MSE 0.043165 versus baseline 0.055977, with paired delta -0.012812 and permutation p=0.0081; all eight paired differences favored the idea. The trained-model mechanism signature was confirmed: mean swap sensitivity was 1.27e-08 for the idea versus 0.045198 for baseline.",
  "metrics": {
    "baseline": "MSE 0.0559766, std 0.0045134; best sweep lr=0.001, epochs=20.",
    "idea": "MSE 0.0431650, std 0.0048839; best sweep lr=0.003, epochs=20; delta=-0.0128116; p=0.0081; 8/8 wins; token-swap sensitivity 1.27e-08 versus baseline 0.0451983."
  },
  "how_to_run": "/home/maxwelhelp/main/bin/python3 run_bench.py",
  "files": [
    "run_bench.py",
    "bench_report.json"
  ],
  "limitations": "Tested one registered correlated-token regression track and one matched baseline architecture; the explicit factorial antisymmetrizer and ODE integration aspects were not benchmarked at neural-network scale. The baseline and idea have similar but not exactly identical parameter counts, and no vision, sequence, or dynamics track was run because those tracks lack the required unordered-token structure.",
  "bench_report": {
    "bench_version": 1,
    "track": "correlated_token_moe_regression",
    "model": "matched_custom",
    "metric_direction": "lower is better",
    "n_seeds": 8,
    "baseline": {
      "best_cfg": {
        "lr": 0.001,
        "epochs": 20
      },
      "sweep": [
        {
          "cfg": {
            "lr": 0.001,
            "epochs": 20
          },
          "mean": 0.05792584456503391
        },
        {
          "cfg": {
            "lr": 0.003,
            "epochs": 20
          },
          "mean": 0.0597206624224782
        },
        {
          "cfg": {
            "lr": 0.01,
            "epochs": 20
          },
          "mean": 0.06360050570219755
        }
      ],
      "full": {
        "mean": 0.05597660783678293,
        "std": 0.004513427280072455,
        "per_seed": [
          0.04968653619289398,
          0.060288313776254654,
          0.06273896247148514,
          0.05898956581950188,
          0.051729243248701096,
          0.05691293999552727,
          0.056976065039634705,
          0.05049123615026474
        ],
        "n": 8
      }
    },
    "idea": {
      "mean": 0.043164961505681276,
      "std": 0.004883880354277062,
      "per_seed": [
        0.03658263012766838,
        0.043850693851709366,
        0.04236328974366188,
        0.04947901517152786,
        0.041660089045763016,
        0.051025427877902985,
        0.04368860274553299,
        0.03666994348168373
      ],
      "n": 8
    },
    "comparison": {
      "delta_mean": -0.012811646331101656,
      "idea_wins": 8,
      "n_pairs": 8,
      "per_seed_diffs": [
        -0.013103906065225601,
        -0.016437619924545288,
        -0.020375672727823257,
        -0.009510550647974014,
        -0.01006915420293808,
        -0.005887512117624283,
        -0.013287462294101715,
        -0.013821292668581009
      ],
      "p_value": 0.0081,
      "mde": 0.0037133339468910474,
      "mde_rel_pct": 6.633724497415811,
      "verdict": "idea better (significant)",
      "system_worked": true
    },
    "mechanism_signature": {
      "quantity": "mean absolute prediction change under swapping two trained-task tokens",
      "prediction": "equivariant/invariant system has zero change while flattened baseline is nonzero",
      "baseline_mean": 0.04519833158701658,
      "idea_mean": 1.2663804227486253e-08,
      "confirmed": true
    },
    "idea_sweep": [
      {
        "cfg": {
          "lr": 0.001,
          "epochs": 20
        },
        "mean": 0.0434781601652503
      },
      {
        "cfg": {
          "lr": 0.003,
          "epochs": 20
        },
        "mean": 0.043164961505681276
      },
      {
        "cfg": {
          "lr": 0.01,
          "epochs": 20
        },
        "mean": 0.04354450246319175
      }
    ],
    "track_justification": "The registered correlated-token track contains exchangeable multi-token correlations; built-in tracks do not.",
    "custom_track": {
      "name": "correlated_token_moe_regression",
      "file": "registered bench custom track",
      "domain": "moe-routing"
    }
  },
  "system_verdict": "worked",
  "practical_verdict": "helps",
  "mechanism_ok": 1,
  "system_judged": true
}