# Стенд-проверка (stage-2) · промт оператора:

(универсальный)

Ответ агента:

{
  "worked": false,
  "confidence": 9,
  "verdict": "Built and ran a custom correlated multi-token denoising benchmark with matched neural denoisers. Fisher-geodesic spacing reduced trained-model stage-error CV from 0.61044 to 0.47896 and confirmed the analytic equal-arc prediction, but test MSE was significantly worse: 0.38846 versus 0.37566, paired delta +0.01280 and permutation p=0.03075. Therefore the mechanism transferred, but the promised task-level improvement did not.",
  "metrics": {
    "baseline": "Test MSE mean 0.3756636306643486 over 8 seeds; best sweep config lr=0.003, smax=1.0.",
    "idea": "Test MSE mean 0.38846133276820183 over 8 seeds; paired delta +0.012797702103853226; permutation p=0.03075; 1/8 paired wins.",
    "mechanism_signature": {
      "predicted_equal_arc_local_kl_cv": 1.7439286709141664e-06,
      "observed_parameter_path_linear_kl_cv": 0.6206565234045026,
      "trained_model_linear_stage_error_cv": 0.610444495224178,
      "trained_model_fisher_stage_error_cv": 0.4789595501048241,
      "confirmed": true
    }
  },
  "how_to_run": "/home/maxwelhelp/main/bin/python3 run_bench.py",
  "files": [
    "fisher_bench_track.py",
    "run_bench.py",
    "bench_report.json"
  ],
  "limitations": "This was a small custom synthetic multitoken denoising task with 400 training examples, 160 test examples, a simple denoiser, 18 epochs, and one schedule length. CIFAR-10 diffusion, pretrained score models, FID, Langevin relaxation variants, larger models, and noisy Fisher estimation were not tested.",
  "bench_report": {
    "bench_version": 1,
    "track": "fisher_multitoken_denoising",
    "model": "custom_denoiser",
    "metric_direction": "lower is better",
    "n_seeds": 8,
    "baseline": {
      "best_cfg": {
        "lr": 0.003,
        "smax": 1.0
      },
      "full": {
        "mean": 0.3756636306643486,
        "std": 0.0827197936263635,
        "per_seed": [
          0.29077786207199097,
          0.32696259021759033,
          0.3145223557949066,
          0.2919705808162689,
          0.4482685327529907,
          0.4811434745788574,
          0.5064208507537842,
          0.34524279832839966
        ]
      }
    },
    "idea": {
      "mean": 0.38846133276820183,
      "std": 0.08522182674569097,
      "per_seed": [
        0.2787860035896301,
        0.3444952070713043,
        0.3325658440589905,
        0.30363890528678894,
        0.4642293453216553,
        0.5044667720794678,
        0.5081693530082703,
        0.37133923172950745
      ]
    },
    "comparison": {
      "delta_mean": 0.012797702103853226,
      "idea_wins": 1,
      "n_pairs": 8,
      "p_value": 0.03075,
      "verdict": "idea worse (significant)",
      "system_worked": false
    },
    "custom_track": {
      "name": "fisher_multitoken_denoising",
      "file": "fisher_bench_track.py",
      "domain": "diffusion-sampling"
    },
    "mechanism_signature": {
      "predicted_equal_arc_local_kl_cv": 1.7439286709141664e-06,
      "observed_parameter_path_linear_kl_cv": 0.6206565234045026,
      "trained_model_stage_behavior": {
        "linear_stage_error_cv": 0.610444495224178,
        "fisher_stage_error_cv": 0.4789595501048241
      },
      "confirmed": true
    }
  },
  "system_verdict": "partial",
  "practical_verdict": "harms",
  "mechanism_ok": 1,
  "system_judged": true
}