# Стенд-проверка (stage-2) · промт оператора:

(универсальный)

Ответ агента:

{
  "worked": false,
  "confidence": 9,
  "verdict": "Built a causal two-layer transformer on the structurally matched sequence forecasting track, comparing an unconstrained learned relative-lag table against an 8-component positive exponential-mixture bias with identical training conditions. The mechanism signature was confirmed from the trained models: maximum bias first difference was -0.04459, while total content-dependent attention was not monotone. However, baseline and idea test MSE were effectively identical (0.2124029454 vs 0.2124029398), with paired delta -5.6e-9 and permutation p=0.5058; therefore there is no observed accuracy win.",
  "metrics": {
    "baseline": "test MSE mean 0.2124029453843832, std 0.029564272137155054; best lr 0.0015; per-seed [0.2271955609, 0.1741580069, 0.2089696825, 0.2423083037, 0.1920005530, 0.1792486161, 0.2660641670, 0.2092786729]",
    "idea": "test MSE mean 0.21240293979644775, std 0.02956426589305894; best lr 0.0015; per-seed [0.2271955609, 0.1741579920, 0.2089696825, 0.2423083037, 0.1920005530, 0.1792486310, 0.2660641382, 0.2092786580]",
    "paired_comparison": {
      "delta_mean": -5.587935447692871e-09,
      "p_value": 0.5058,
      "idea_wins": 3,
      "n_pairs": 8,
      "verdict": "no measurable effect"
    },
    "mechanism_signature": {
      "prediction": "trained mixture log-kernel bias decreases with lag; retained content logits can make total attention nonmonotone",
      "observed_bias_max_first_difference": -0.044586181640625,
      "observed_bias_min_second_difference": 0.0005350112915039062,
      "observed_attention_max_first_difference": 0.030105771829364122,
      "n_models": 8,
      "confirmed": true
    }
  },
  "bench_report": {
    "bench_version": 1,
    "track": "sequence",
    "model": "transformer_tiny",
    "metric_direction": "lower is better",
    "n_seeds": 8,
    "baseline": {
      "best_cfg": {
        "lr": 0.0015,
        "gamma": 1.0
      },
      "sweep": [
        {
          "cfg": {
            "lr": 0.0015,
            "gamma": 1.0
          },
          "mean": 0.21315788850188255
        },
        {
          "cfg": {
            "lr": 0.003,
            "gamma": 1.0
          },
          "mean": 0.2156946398317814
        },
        {
          "cfg": {
            "lr": 0.006,
            "gamma": 1.0
          },
          "mean": 0.25452903658151627
        }
      ],
      "full": {
        "mean": 0.2124029453843832,
        "std": 0.029564272137155054,
        "per_seed": [
          0.22719556093215942,
          0.1741580069065094,
          0.20896968245506287,
          0.24230830371379852,
          0.19200055301189423,
          0.1792486160993576,
          0.2660641670227051,
          0.2092786729335785
        ],
        "n": 8
      }
    },
    "idea": {
      "mean": 0.21240293979644775,
      "std": 0.02956426589305894,
      "per_seed": [
        0.22719556093215942,
        0.1741579920053482,
        0.20896968245506287,
        0.24230830371379852,
        0.19200055301189423,
        0.1792486310005188,
        0.2660641372203827,
        0.2092786580324173
      ],
      "n": 8
    },
    "comparison": {
      "delta_mean": -5.587935447692871e-09,
      "idea_wins": 3,
      "n_pairs": 8,
      "per_seed_diffs": [
        0.0,
        -1.4901161193847656e-08,
        0.0,
        0.0,
        0.0,
        1.4901161193847656e-08,
        -2.9802322387695312e-08,
        -1.4901161193847656e-08
      ],
      "p_value": 0.5058,
      "mde": 1.1414612684177589e-08,
      "mde_rel_pct": 5.374036910609076e-06,
      "verdict": "no measurable effect",
      "system_worked": false
    },
    "protocol_notes": "Matched sequence track; shared 2-block d=64 causal transformer, differing only in relative lag bias: unconstrained table versus 8-positive-exponential mixture.",
    "mechanism_signature": {
      "prediction": "trained mixture log-kernel bias decreases with lag; retained content logits can make total attention nonmonotone",
      "observed_bias_max_first_difference": -0.044586181640625,
      "observed_bias_min_second_difference": 0.0005350112915039062,
      "observed_attention_max_first_difference": 0.030105771829364122,
      "n_models": 8,
      "confirmed": true
    },
    "idea_sweep": [
      {
        "cfg": {
          "lr": 0.0015,
          "gamma": 1.0
        },
        "mean": 0.21315788477659225
      },
      {
        "cfg": {
          "lr": 0.003,
          "gamma": 1.0
        },
        "mean": 0.21569465845823288
      },
      {
        "cfg": {
          "lr": 0.006,
          "gamma": 1.0
        },
        "mean": 0.25452912598848343
      }
    ],
    "idea_config": {
      "lr": 0.0015,
      "gamma": 1.0
    },
    "search_space": {
      "baseline_grid": [
        {
          "lr": 0.0015,
          "gamma": 1.0
        },
        {
          "lr": 0.003,
          "gamma": 1.0
        },
        {
          "lr": 0.006,
          "gamma": 1.0
        }
      ],
      "idea_grid": [
        {
          "lr": 0.0015,
          "gamma": 1.0
        },
        {
          "lr": 0.003,
          "gamma": 1.0
        },
        {
          "lr": 0.006,
          "gamma": 1.0
        }
      ]
    }
  },
  "how_to_run": "/home/maxwelhelp/main/bin/python3 stage2_bench.py",
  "files": [
    "stage2_bench.py",
    "bench_report.json"
  ],
  "limitations": "Only the built-in sequence forecasting track was tested; no CIFAR, dynamics, language modeling, length extrapolation, ALiBi/RoPE comparison, runtime benchmark, or larger-context evaluation was performed. The benchmark used 800 training examples, 300 test examples, 10 epochs, and three learning rates; the baseline decisive bias-strength knob was not independently swept because the canonical baseline was the learned lag table itself and the shared gamma was fixed at 1.0.",
  "system_verdict": "partial",
  "practical_verdict": "no_effect",
  "mechanism_ok": 1,
  "system_judged": true
}