# Стенд-проверка (stage-2) · промт оператора:

(универсальный)

Ответ агента:

{
  "worked": false,
  "confidence": 10,
  "verdict": "Built a local exact-orbit mask-search benchmark using the bench tabular track and a shared six-channel MLP, with a three-value learning-rate sweep on both baseline and idea sides and eight paired seeds. Canonical search reduced evaluated candidates from 20 to 1, but the idea's test MSE was 208.203 versus 185.918 for exhaustive baseline (paired delta +22.285, permutation p=0.0081), and trained-model mask spreads were nonzero for all eight seeds. The structural speedup is real, but exact channel symmetry was not present in independently trained hidden units, so the promised neural-training benefit did not transfer.",
  "metrics": {
    "baseline": "Best lr=0.01; 20 masks; full 8-seed MSE mean 185.9184, std 13.3165",
    "idea": "Best lr=0.01; 1 canonical mask; full 8-seed MSE mean 208.2031, std 13.8050",
    "paired_delta": "idea - baseline = +22.2847 MSE; idea wins 0/8; permutation p=0.0081",
    "evaluation_reduction": "20x fewer mask evaluations",
    "mechanism_signature": "Predicted one orbit and 20x reduction; observed reduction 20x, but trained mask metric range was 36.05-73.77 across seeds and nonzero for 8/8, confirmed=false"
  },
  "bench_report": {
    "bench_version": 1,
    "track": "tabular",
    "model": "masked_mlp_tiny",
    "metric_direction": "lower is better",
    "n_seeds": 8,
    "baseline": {
      "best_cfg": {
        "lr": 0.01,
        "epochs": 4,
        "mask_count": 20
      },
      "sweep": [
        {
          "cfg": {
            "lr": 0.001,
            "epochs": 4,
            "mask_count": 20
          },
          "mean": 233.01842498779297
        },
        {
          "cfg": {
            "lr": 0.003,
            "epochs": 4,
            "mask_count": 20
          },
          "mean": 228.20503997802734
        },
        {
          "cfg": {
            "lr": 0.01,
            "epochs": 4,
            "mask_count": 20
          },
          "mean": 191.4046401977539
        }
      ],
      "full": {
        "mean": 185.9184112548828,
        "std": 13.31649292644178,
        "per_seed": [
          211.56298828125,
          186.1365509033203,
          195.3025665283203,
          172.616455078125,
          170.3512420654297,
          193.65509033203125,
          172.0006561279297,
          185.72174072265625
        ],
        "n": 8
      }
    },
    "idea": {
      "mean": 208.20309448242188,
      "std": 13.805027643998322,
      "per_seed": [
        226.8892364501953,
        212.61376953125,
        219.84451293945312,
        193.2020263671875,
        220.1836700439453,
        202.975341796875,
        183.25924682617188,
        206.65695190429688
      ],
      "n": 8
    },
    "comparison": {
      "delta_mean": 22.284683227539062,
      "idea_wins": 0,
      "n_pairs": 8,
      "per_seed_diffs": [
        15.326248168945312,
        26.477218627929688,
        24.5419462890625,
        20.5855712890625,
        49.832427978515625,
        9.32025146484375,
        11.258590698242188,
        20.935211181640625
      ],
      "p_value": 0.0081,
      "mde": 10.597026350103498,
      "mde_rel_pct": 5.699826218703871,
      "verdict": "idea worse (significant)",
      "system_worked": false
    },
    "idea_sweep": [
      {
        "cfg": {
          "lr": 0.001,
          "epochs": 4,
          "mask_count": 1,
          "canonical": true
        },
        "mean": 232.431640625
      },
      {
        "cfg": {
          "lr": 0.003,
          "epochs": 4,
          "mask_count": 1,
          "canonical": true
        },
        "mean": 228.43239402770996
      },
      {
        "cfg": {
          "lr": 0.01,
          "epochs": 4,
          "mask_count": 1,
          "canonical": true
        },
        "mean": 208.20309448242188
      }
    ],
    "runtime_sec": 36.57714536000003,
    "track_justification": "Tabular is the built-in architecture/regularization-adjacent track; the experiment directly operates on exchangeable hidden channels and structured pruning masks.",
    "mask_definition": "retain k=3 of six first-layer hidden channels; exact S_6 symmetry represented by channel permutations.",
    "mechanism_signature": {
      "predicted_orbits": 1,
      "observed_candidate_masks": 20,
      "predicted_evaluation_reduction": 20,
      "observed_evaluation_reduction": 20,
      "trained_model_mask_spread": [
        {
          "seed": 0,
          "mask_metric_std": 18.708847715759166,
          "mask_metric_range": 73.77125549316406
        },
        {
          "seed": 1,
          "mask_metric_std": 11.349517459757626,
          "mask_metric_range": 53.40818786621094
        },
        {
          "seed": 2,
          "mask_metric_std": 13.9794193204962,
          "mask_metric_range": 43.052520751953125
        },
        {
          "seed": 3,
          "mask_metric_std": 9.279172900445374,
          "mask_metric_range": 36.04695129394531
        },
        {
          "seed": 4,
          "mask_metric_std": 14.554493291709218,
          "mask_metric_range": 54.18971252441406
        },
        {
          "seed": 5,
          "mask_metric_std": 12.235878564511857,
          "mask_metric_range": 44.35304260253906
        },
        {
          "seed": 6,
          "mask_metric_std": 12.367370327709848,
          "mask_metric_range": 58.37591552734375
        },
        {
          "seed": 7,
          "mask_metric_std": 18.78608365872588,
          "mask_metric_range": 67.00553894042969
        }
      ],
      "observed_nonzero_spread_seeds": 8,
      "confirmed": false,
      "note": "The exact reduction is structural; finite independently trained masked models can differ because optimization noise breaks numerical equality."
    }
  },
  "how_to_run": "/home/maxwelhelp/main/bin/python3 bench_experiment.py",
  "files": [
    "bench_experiment.py",
    "bench_report.json"
  ],
  "limitations": "Only the tabular track was tested; no CNN, MoE, multi-block symmetry, stabilizer-chain backend, or exact weight-permutation-equivalent initialization/training protocol was tested. The experiment demonstrates that naive index-level channel exchangeability is not an exact symmetry of independently trained networks.",
  "system_verdict": "failed",
  "practical_verdict": "harms",
  "mechanism_ok": 0,
  "system_judged": true
}