# Стенд-проверка (stage-2) · промт оператора:

(универсальный)

Ответ агента:

{
  "worked": false,
  "confidence": 10,
  "verdict": "Built a local stage-2 implementation of task-visible axis pruning for the structurally matched tabular ReLU MLP track, using the benchmark training path, equal learning-rate sweeps, eight paired seeds, and post-training fine-tuning. Magnitude pruning achieved test MSE 7.6000 versus 8.1544 for task-visible pruning, with paired delta +0.5544 and permutation p=0.0156, so the idea was significantly worse. The model-derived mechanism signature also failed: the observed maximum replacement error was 0.1555 rather than the predicted zero, so there is no evidence of the promised effect on this benchmark.",
  "metrics": {
    "baseline": "Magnitude pruning at 50% hidden width; best lr=0.01; full 8-seed mean test MSE 7.599956, std 0.805135.",
    "idea": "Task-visible quantile crossing pruning at 50% hidden width; best lr=0.01; full 8-seed mean test MSE 8.154385, std 1.038946; paired delta +0.554429; 1/8 wins; permutation p=0.0156.",
    "mechanism_signature": "53 trained-model one-sided selected axes tested; predicted maximum replacement error 0.0, observed maximum error 0.155509, confirmed=false."
  },
  "how_to_run": "/home/maxwelhelp/main/bin/python3 stage2_bench.py",
  "files": [
    "stage2_bench.py",
    "bench_report.json"
  ],
  "bench_report": {
    "bench_version": 1,
    "track": "tabular",
    "model": "mlp_tiny",
    "metric_direction": "lower is better",
    "n_seeds": 8,
    "baseline": {
      "best_cfg": {
        "lr": 0.01
      },
      "sweep": [
        {
          "cfg": {
            "lr": 0.001
          },
          "mean": 16.10005497932434
        },
        {
          "cfg": {
            "lr": 0.003
          },
          "mean": 11.262676000595093
        },
        {
          "cfg": {
            "lr": 0.01
          },
          "mean": 7.429885745048523
        }
      ],
      "full": {
        "mean": 7.599956035614014,
        "std": 0.8051347687842139,
        "per_seed": [
          6.844658851623535,
          6.109804630279541,
          8.30047607421875,
          8.464603424072266,
          7.705488681793213,
          6.993262767745971,
          8.389989852905273,
          7.991364002227783
        ],
        "n": 8
      }
    },
    "idea": {
      "mean": 8.154384911060333,
      "std": 1.0389464855334478,
      "per_seed": [
        7.118060111999512,
        6.544161319732666,
        9.722457885742188,
        8.549818992614746,
        7.820003986358643,
        7.930368423461914,
        9.617923736572266,
        7.932284832000732
      ],
      "n": 8
    },
    "comparison": {
      "delta_mean": 0.5544288754463196,
      "idea_wins": 1,
      "n_pairs": 8,
      "per_seed_diffs": [
        0.27340126037597656,
        0.434356689453125,
        1.4219818115234375,
        0.08521556854248047,
        0.11451530456542969,
        0.937105655670166,
        1.2279338836669922,
        -0.05907917022705078
      ],
      "p_value": 0.0156,
      "mde": 0.4725954183814587,
      "mde_rel_pct": 6.218396740281628,
      "verdict": "idea worse (significant)",
      "system_worked": false
    },
    "track_justification": "ReLU hidden-axis pruning is structurally matched to the tabular MLP track; both systems use the identical trained mlp_tiny and optimizer budgets.",
    "pruning_fraction": 0.5,
    "idea_config": {
      "lr": 0.01
    },
    "idea_sweep": [
      {
        "cfg": {
          "lr": 0.001
        },
        "result": {
          "mean": 22.81420075893402,
          "std": 5.572511935733092,
          "per_seed": [
            23.423412322998047,
            15.931241035461426,
            33.59147262573242,
            17.00380516052246,
            26.614585876464844,
            17.325483322143555,
            25.237001419067383,
            23.38660430908203
          ],
          "n": 8
        }
      },
      {
        "cfg": {
          "lr": 0.003
        },
        "result": {
          "mean": 14.472419738769531,
          "std": 2.867801864411425,
          "per_seed": [
            12.962059020996094,
            10.092577934265137,
            17.148317337036133,
            11.091870307922363,
            15.618391036987305,
            13.338394165039062,
            16.90511131286621,
            18.622636795043945
          ],
          "n": 8
        }
      },
      {
        "cfg": {
          "lr": 0.01
        },
        "result": {
          "mean": 8.154384911060333,
          "std": 1.0389464855334478,
          "per_seed": [
            7.118060111999512,
            6.544161319732666,
            9.722457885742188,
            8.549818992614746,
            7.820003986358643,
            7.930368423461914,
            9.617923736572266,
            7.932284832000732
          ],
          "n": 8
        }
      }
    ],
    "parameter_fraction_estimate": 0.25,
    "mechanism_signature": {
      "prediction": "strictly one-sided selected ReLU axes are affine or zero on the test patch",
      "predicted_max_replacement_error": 0.0,
      "observed_max_replacement_error": 0.1555086076259613,
      "tested_one_sided_axes": 53,
      "downstream_norm_sum": 24.949480190873146,
      "confirmed": false
    }
  },
  "limitations": "Only the tabular track and one 50% retained-width setting were tested; physical compact-layer reconstruction and measured wall-clock/FLOP speedups were not benchmarked. The benchmark's quantile criterion is empirical rather than an exact all-input strict sign certificate, and the mechanism signature consequently tests behavior on a finite held-out patch.",
  "system_verdict": "failed",
  "practical_verdict": "harms",
  "mechanism_ok": 0,
  "system_judged": true
}