# Стенд-проверка (stage-2) · промт оператора:

(универсальный)

Ответ агента:

{
  "worked": false,
  "confidence": 9,
  "verdict": "Built an end-to-end matched frozen-neural-dictionary benchmark on the structurally appropriate manufactured_heat_pde track, comparing raw coefficients with quadrature-whitened coefficients under the canonical train_model loop. The mass identity prediction was confirmed from the trained benchmark system (error 3.70e-14), but test MSE was substantially worse for the idea (1.3856 versus 0.07675), with paired delta +1.3089 and permutation p=0.0081. Therefore the mechanism works algebraically but does not yield the promised benchmark improvement.",
  "metrics": {
    "baseline": "Best baseline lr=0.003, ridge=0.0; full 8-seed test MSE mean=0.07674595, std=0.00499025; per-seed=[0.08141968,0.06942547,0.07098477,0.08333057,0.08016344,0.07751243,0.07957722,0.07155404].",
    "idea": "Best idea setting lr=0.01, tau=0.001, ridge=0.0; full 8-seed test MSE mean=1.38563919, std=1.91728933; per-seed=[0.57850820,0.35083449,0.25561139,1.90971708,6.16149235,1.61558259,0.12886217,0.08450527].",
    "paired_comparison": "delta_mean=+1.30889324 (positive means idea worse), idea_wins=0/8, permutation_p=0.0081, verdict='idea worse (significant)'",
    "mechanism_signature": "assembly_mass_identity_error=3.70488796e-14; trained_raw_prediction_std=0.28439134; trained_whitened_prediction_std=0.58684784; confirmed=true"
  },
  "bench_report": {
    "bench_version": 1,
    "track": "manufactured_heat_pde",
    "model": "mlp_tiny",
    "metric_direction": "lower is better",
    "n_seeds": 8,
    "baseline": {
      "best_cfg": {
        "lr": 0.003,
        "ridge": 0.0
      },
      "sweep": [
        {
          "cfg": {
            "lr": 0.001,
            "ridge": 0.0
          },
          "mean": 0.08385927602648735
        },
        {
          "cfg": {
            "lr": 0.003,
            "ridge": 0.0
          },
          "mean": 0.07629012316465378
        },
        {
          "cfg": {
            "lr": 0.01,
            "ridge": 0.0
          },
          "mean": 0.07660810463130474
        },
        {
          "cfg": {
            "lr": 0.003,
            "ridge": 0.001
          },
          "mean": 0.0762928407639265
        }
      ],
      "full": {
        "mean": 0.0767459524795413,
        "std": 0.004990245344286674,
        "per_seed": [
          0.0814196839928627,
          0.06942547112703323,
          0.07098476588726044,
          0.08333057165145874,
          0.08016344159841537,
          0.07751242816448212,
          0.07957721501588821,
          0.0715540423989296
        ],
        "n": 8
      }
    },
    "idea": {
      "mean": 1.3856391916051507,
      "std": 1.9172893293623836,
      "per_seed": [
        0.578508198261261,
        0.3508344888687134,
        0.255611389875412,
        1.909717082977295,
        6.161492347717285,
        1.6155825853347778,
        0.1288621723651886,
        0.08450526744127274
      ],
      "n": 8
    },
    "comparison": {
      "delta_mean": 1.3088932391256094,
      "idea_wins": 0,
      "n_pairs": 8,
      "per_seed_diffs": [
        0.4970885142683983,
        0.28140901774168015,
        0.18462662398815155,
        1.8263865113258362,
        6.08132890611887,
        1.5380701571702957,
        0.049284957349300385,
        0.01295122504234314
      ],
      "p_value": 0.0081,
      "mde": 1.7118928334633723,
      "mde_rel_pct": 2230.5968955427625,
      "verdict": "idea worse (significant)",
      "system_worked": false
    },
    "mechanism_signature": {
      "mechanism_signature": {
        "assembly_mass_identity_error": 3.704887957203547e-14,
        "trained_raw_prediction_std": 0.2843913435935974,
        "trained_whitened_prediction_std": 0.5868478417396545,
        "predicted_identity_error_lt_1e-8": true,
        "confirmed": true
      },
      "idea_sweep": [
        {
          "cfg": {
            "lr": 0.001,
            "ridge": 0.0
          },
          "result": {
            "mean": 150.65098530054092,
            "std": 170.37985075799597,
            "per_seed": [
              0.8372825384140015,
              208.86212158203125,
              1.252848505973816,
              474.57147216796875,
              332.5594482421875,
              185.36181640625,
              0.778637170791626,
              0.9842557907104492
            ],
            "n": 8
          }
        },
        {
          "cfg": {
            "lr": 0.003,
            "ridge": 0.0
          },
          "result": {
            "mean": 8.145590918138623,
            "std": 12.121363833401968,
            "per_seed": [
              0.7712369561195374,
              3.3300294876098633,
              0.5697437524795532,
              34.8777961730957,
              21.517240524291992,
              3.837894916534424,
              0.14831362664699554,
              0.11247190833091736
            ],
            "n": 8
          }
        },
        {
          "cfg": {
            "lr": 0.01,
            "ridge": 0.0
          },
          "result": {
            "mean": 1.3856391916051507,
            "std": 1.9172893293623836,
            "per_seed": [
              0.578508198261261,
              0.3508344888687134,
              0.255611389875412,
              1.909717082977295,
              6.161492347717285,
              1.6155825853347778,
              0.1288621723651886,
              0.08450526744127274
            ],
            "n": 8
          }
        }
      ],
      "matched_tau": 0.001
    },
    "custom_track": null
  },
  "how_to_run": "/home/maxwelhelp/main/bin/python3 bench_experiment.py",
  "files": [
    "bench_experiment.py",
    "bench_report.json"
  ],
  "limitations": "Only the manufactured 1D parabolic PDE track was tested; no larger spatial PDE, multi-dimensional domain, nonlinear PDE, wall-clock/FLOP comparison, or alternative whitening cutoffs in the final full paired protocol were evaluated. The implementation uses a frozen tanh dictionary with a trainable linear readout rather than a jointly trained deep spatial feature network, and the benchmark's ridge sweep is implemented as Adam weight decay.",
  "system_verdict": "failed",
  "practical_verdict": "harms",
  "mechanism_ok": 0,
  "system_judged": true
}