# Стенд-проверка (stage-2) · промт оператора:

(универсальный)

Ответ агента:

{
  "worked": true,
  "confidence": 7,
  "verdict": "The registered custom graph-landmark forecasting track was evaluated with identical transformer systems, differing only by the addition of greedy resolving-landmark signatures. The idea achieved lower test MSE on all 8 paired seeds: 0.066688 versus the tuned baseline 0.097288, paired delta -0.030600, permutation p=0.0081. The trained-model mechanism check observed zero idea signature collisions versus 19 baseline collisions, below the independently computed hypergeometric upper bound of 114.",
  "metrics": {
    "baseline": "make_report baseline block: best_cfg lr=0.003; sweep means lr=0.001: 0.0976012, lr=0.003: 0.0971596, lr=0.01: 0.0978763; full 8-seed mean=0.0972881, std=0.0028138.",
    "idea": "Best idea config lr=0.01; per-seed test MSE=[0.0671466,0.0738764,0.0655859,0.0592485,0.0516633,0.0640472,0.0735524,0.0783868], mean=0.0666884, std=0.0081020.",
    "comparison": "delta_mean=-0.0305997, idea_wins=8, n_pairs=8, p_value=0.0081, verdict='idea better (significant)', system_worked=true"
  },
  "bench_report": {
    "bench_version": 1,
    "track": "graph_landmark_forecast",
    "model": "transformer_tiny",
    "metric_direction": "lower is better",
    "n_seeds": 8,
    "baseline": {
      "best_cfg": {
        "lr": 0.003
      },
      "sweep": [
        {
          "cfg": {
            "lr": 0.001
          },
          "mean": 0.09760121069848537
        },
        {
          "cfg": {
            "lr": 0.003
          },
          "mean": 0.09715960919857025
        },
        {
          "cfg": {
            "lr": 0.01
          },
          "mean": 0.09787633270025253
        }
      ],
      "full": {
        "mean": 0.09728806372731924,
        "std": 0.0028138072522287676,
        "per_seed": [
          0.0970306470990181,
          0.09655177593231201,
          0.09898804873228073,
          0.09606796503067017,
          0.10214304178953171,
          0.09961570054292679,
          0.09206581115722656,
          0.09584151953458786
        ],
        "n": 8
      },
      "full_grid": [
        {
          "cfg": {
            "lr": 0.001
          },
          "result": {
            "mean": 0.09813422616571188,
            "std": 0.003308826432344093,
            "per_seed": [
              0.09822424501180649,
              0.09727376699447632,
              0.0995187908411026,
              0.09538803994655609,
              0.10483681410551071,
              0.10019651055335999,
              0.09305645525455475,
              0.0965791866183281
            ],
            "n": 8
          }
        },
        {
          "cfg": {
            "lr": 0.003
          },
          "result": {
            "mean": 0.09728806372731924,
            "std": 0.0028138072522287676,
            "per_seed": [
              0.0970306470990181,
              0.09655177593231201,
              0.09898804873228073,
              0.09606796503067017,
              0.10214304178953171,
              0.09961570054292679,
              0.09206581115722656,
              0.09584151953458786
            ],
            "n": 8
          }
        },
        {
          "cfg": {
            "lr": 0.01
          },
          "result": {
            "mean": 0.09821864496916533,
            "std": 0.002263958786368283,
            "per_seed": [
              0.09782364219427109,
              0.09844150394201279,
              0.09902690351009369,
              0.09621328115463257,
              0.10131707042455673,
              0.10156738013029099,
              0.09458042681217194,
              0.09677895158529282
            ],
            "n": 8
          }
        }
      ]
    },
    "idea": {
      "mean": 0.06668837741017342,
      "std": 0.00810202264282473,
      "per_seed": [
        0.06714659929275513,
        0.07387638092041016,
        0.06558588147163391,
        0.059248507022857666,
        0.05166328698396683,
        0.06404716521501541,
        0.07355235517024994,
        0.07838684320449829
      ],
      "n": 8
    },
    "comparison": {
      "delta_mean": -0.030599686317145824,
      "idea_wins": 8,
      "n_pairs": 8,
      "per_seed_diffs": [
        -0.02988404780626297,
        -0.022675395011901855,
        -0.03340216726064682,
        -0.0368194580078125,
        -0.05047975480556488,
        -0.03556853532791138,
        -0.018513455986976624,
        -0.01745467633008957
      ],
      "p_value": 0.0081,
      "mde": 0.00920648763417379,
      "mde_rel_pct": 9.463121457507778,
      "verdict": "idea better (significant)",
      "system_worked": true
    },
    "custom_track": {
      "name": "graph_landmark_forecast",
      "file": "graph_landmark_track.py",
      "domain": "graph-nn"
    },
    "mechanism_signature": {
      "prediction": "expected collisions are bounded by the hypergeometric union bound",
      "c_min": 1,
      "s": 8,
      "predicted_collision_upper_bound": 114.0,
      "observed_idea_collision_count": 0.0,
      "baseline_collision_count": 19.0,
      "confirmed": true
    }
  },
  "how_to_run": "cd /home/maxwelhelp/all/math2nn/experiments/exp421_resolving-landmark-bottleneck && /home/maxwelhelp/main/bin/python3 stage2_bench.py",
  "files": [
    "graph_landmark_track.py",
    "stage2_bench.py",
    "bench_report.json",
    "bench_stdout.txt",
    "bench_stdout_registered.txt"
  ],
  "limitations": "The registered custom task is small: a fixed 20-node graph with 400 training and 200 test examples. Large-graph memory, preprocessing, and throughput scaling were not measured. The baseline is a matched transformer with zero structural channels rather than a production full Graphormer adjacency-attention implementation, so the significant MSE win supports the feature mechanism on this graph task but does not by itself establish the claimed O(n^2) to O(ns) systems-level memory reduction. The mechanism bound is loose because c_min=1.",
  "system_verdict": "worked",
  "practical_verdict": "helps",
  "mechanism_ok": 1,
  "system_judged": true
}