# Стенд-проверка (stage-2) · промт оператора:

(универсальный)

Ответ агента:

{
  "worked": false,
  "confidence": 10,
  "verdict": "Built a local transformer sequence benchmark with a fixed connected spanning-tree token support as the intervention, using identical data, model dimensions, optimizer, epochs, batch size, and shared learning-rate sweep for baseline and idea. The mechanism signature was confirmed at NN scale: 992 dense directed token edges versus 62 tree-directed edges, a 16x reduction, with maximum support degree 2.0. However, the idea had test MSE 0.43197 versus baseline 0.39103, paired delta +0.04093, and permutation p=0.0081; it was significantly worse, so worked=false.",
  "metrics": {
    "baseline": "Sequence transformer_tiny, 8-seed test MSE mean 0.3910339922; best lr=0.001 selected by 4-seed sweep.",
    "idea": "Resistance-certified tree-support transformer, 8-seed test MSE mean 0.4319681115 at best idea lr=0.001; paired delta +0.0409341194, 0/8 wins, permutation p=0.0081.",
    "edge_behavior": "Dense directed token edges 992; tree directed token edges 62; reduction 16.0x; maximum trained support degree 2.0; mechanism confirmed=true."
  },
  "how_to_run": "/home/maxwelhelp/main/bin/python3 bench_experiment.py",
  "files": [
    "bench_experiment.py",
    "bench_report.json",
    "bench_stdout.txt"
  ],
  "limitations": "Only the built-in sequence forecast track was tested, using a fixed path spanning tree rather than learned conductances, exact effective-resistance computation, Theta projection, or stochastic spanning-tree sampling. The implementation adds a sparse path residual alongside the standard TransformerEncoder rather than replacing the encoder's internal dense self-attention, so the measured edge reduction reflects the explicit intervention support and not a fully sparse Transformer kernel. No Cora/PubMed graph node-classification, GPU speed benchmark, large-graph resistance estimation, or memory measurement was performed.",
  "bench_report": {
    "bench_version": 1,
    "track": "sequence",
    "model": "transformer_tiny",
    "metric_direction": "lower is better",
    "n_seeds": 8,
    "baseline": {
      "best_cfg": {
        "lr": 0.001
      },
      "sweep": [
        {
          "cfg": {
            "lr": 0.001
          },
          "mean": 0.3540426418185234
        },
        {
          "cfg": {
            "lr": 0.003
          },
          "mean": 0.44857239723205566
        },
        {
          "cfg": {
            "lr": 0.01
          },
          "mean": 1.1356653422117233
        }
      ],
      "full": {
        "mean": 0.39103399217128754,
        "std": 0.052424765875202554,
        "per_seed": [
          0.3786984086036682,
          0.2985808253288269,
          0.3361116051673889,
          0.4027797281742096,
          0.4445543587207794,
          0.4033418297767639,
          0.4753662645816803,
          0.38883891701698303
        ],
        "n": 8
      }
    },
    "idea": {
      "mean": 0.4319681115448475,
      "std": 0.05391058937246612,
      "per_seed": [
        0.4073769748210907,
        0.3430413603782654,
        0.3716563582420349,
        0.43455326557159424,
        0.4995949864387512,
        0.4727206230163574,
        0.501372754573822,
        0.425428569316864
      ],
      "n": 8,
      "sweep": [
        {
          "cfg": {
            "lr": 0.001
          },
          "mean": 0.3891569897532463,
          "per_seed": [
            0.4073769748210907,
            0.3430413603782654,
            0.3716563582420349,
            0.43455326557159424
          ]
        },
        {
          "cfg": {
            "lr": 0.003
          },
          "mean": 0.4779898002743721,
          "per_seed": [
            0.4281887710094452,
            0.4741615653038025,
            0.5188824534416199,
            0.49072641134262085
          ]
        },
        {
          "cfg": {
            "lr": 0.01
          },
          "mean": 1.1991955637931824,
          "per_seed": [
            1.1704474687576294,
            0.904765248298645,
            1.6207839250564575,
            1.1007856130599976
          ]
        }
      ],
      "selected_cfg": {
        "lr": 0.001
      }
    },
    "comparison": {
      "delta_mean": 0.04093411937355995,
      "idea_wins": 0,
      "n_pairs": 8,
      "per_seed_diffs": [
        0.028678566217422485,
        0.04446053504943848,
        0.035544753074645996,
        0.031773537397384644,
        0.0550406277179718,
        0.0693787932395935,
        0.026006489992141724,
        0.03658965229988098
      ],
      "p_value": 0.0081,
      "mde": 0.012343224156297255,
      "mde_rel_pct": 3.1565604022707214,
      "verdict": "idea worse (significant)",
      "system_worked": false
    },
    "mechanism_signature": {
      "prediction": "A spanning-tree support should reduce token edge messages from 496 undirected non-self token pairs to 31 undirected path edges (992 versus 62 directed), with maximum expected degree <=2.",
      "observed": {
        "dense_directed_edges": 992,
        "tree_directed_edges": 62,
        "edge_reduction": 16.0,
        "max_expected_degree": 2.0,
        "trained_model_parameter_count": 71169
      },
      "confirmed": true
    }
  },
  "system_verdict": "partial",
  "practical_verdict": "harms",
  "mechanism_ok": 1,
  "system_judged": true
}