# Стенд-проверка (stage-2) · промт оператора:

(универсальный)

Ответ агента:

{
  "worked": true,
  "confidence": 8,
  "verdict": "Built a structurally matched custom directed-path graph regression track and trained additive versus ordered matrix-product systems with identical encoder/readout capacity. Across 8 paired seeds, Path-Holonomy achieved lower test MSE (0.089526 vs 0.191343), with delta -0.101817 and paired permutation p=0.03805, satisfying the benchmark significant-win criterion. The trained-model mechanism signature was measured but not confirmed, so the conclusion rests on the independent task metric.",
  "metrics": {
    "baseline": "best cfg lr=0.01, epochs=35, width=48; 8-seed mean test MSE 0.191343, std 0.012652",
    "idea": "best cfg lr=0.01, epochs=35, width=48; 8-seed mean test MSE 0.089526, std 0.105722",
    "comparison": "delta_mean=-0.101817; idea wins 6/8; permutation p=0.03805",
    "mechanism_signature": "baseline RMSE 0.437195; idea RMSE 0.236979; trained-model reversal order sensitivity 0.368372; confirmed=false"
  },
  "how_to_run": "/home/maxwelhelp/main/bin/python3 run_bench.py",
  "files": [
    "holonomy_track.py",
    "run_bench.py",
    "bench_report.json"
  ],
  "limitations": "The built-in tracks were structurally mismatched, so the required local custom track was used. The task is synthetic rather than molecular or knowledge-graph data; throughput/FLOPs, cycle-consistency loss, learned reversal/color-switch maps, and broader orientation robustness were not tested. The mechanism signature threshold was not confirmed quantitatively.",
  "bench_report": {
    "bench_version": 1,
    "track": "directed_path_holonomy",
    "model": "matched_matrix_mlp",
    "metric_direction": "lower is better",
    "n_seeds": 8,
    "baseline": {
      "best_cfg": {
        "lr": 0.01,
        "epochs": 35,
        "width": 48
      },
      "full": {
        "mean": 0.1913430094718933,
        "std": 0.01265175674434216,
        "n": 8,
        "per_seed": [
          0.21796821057796478,
          0.18948782980442047,
          0.18038326501846313,
          0.17539961636066437,
          0.2016671746969223,
          0.19406883418560028,
          0.18906868994235992,
          0.18270045518875122
        ]
      },
      "sweep": [
        {
          "cfg": {
            "lr": 0.001,
            "epochs": 35,
            "width": 48
          },
          "mean": 0.2922794222831726
        },
        {
          "cfg": {
            "lr": 0.003,
            "epochs": 35,
            "width": 48
          },
          "mean": 0.2332044392824173
        },
        {
          "cfg": {
            "lr": 0.01,
            "epochs": 35,
            "width": 48
          },
          "mean": 0.1908097304403782
        }
      ]
    },
    "idea": {
      "mean": 0.08952617183967959,
      "std": 0.10572229702516253,
      "n": 8,
      "per_seed": [
        0.07830779999494553,
        0.0012882769806310534,
        0.02554335445165634,
        0.0027228575199842453,
        0.2705438435077667,
        0.26158156991004944,
        0.07142530381679535,
        0.004796368535608053
      ]
    },
    "idea_best_cfg": {
      "lr": 0.01,
      "epochs": 35,
      "width": 48
    },
    "idea_sweep": [
      {
        "cfg": {
          "lr": 0.001,
          "epochs": 35,
          "width": 48
        },
        "mean": 0.33307167142629623
      },
      {
        "cfg": {
          "lr": 0.003,
          "epochs": 35,
          "width": 48
        },
        "mean": 0.1860583908855915
      },
      {
        "cfg": {
          "lr": 0.01,
          "epochs": 35,
          "width": 48
        },
        "mean": 0.02696557223680429
      }
    ],
    "comparison": {
      "delta_mean": -0.10181683763221372,
      "idea_wins": 6,
      "n_pairs": 8,
      "per_seed_diffs": [
        -0.13966041058301926,
        -0.18819955282378942,
        -0.1548399105668068,
        -0.17267675884068012,
        0.06887666881084442,
        0.06751273572444916,
        -0.11764338612556458,
        -0.17790408665314317
      ],
      "mde": 0.08970938035328954,
      "mde_rel_pct": 46.88406469663429,
      "p_value": 0.03805,
      "verdict": "idea better (significant)",
      "system_worked": true
    },
    "custom_track": {
      "name": "directed_path_holonomy",
      "file": "holonomy_track.py",
      "domain": "graph"
    },
    "mechanism_signature": {
      "prediction": "ordered holonomy should retain order: reversal sensitivity exceeds additive baseline gap",
      "observed_mean": {
        "baseline_test_rmse": 0.4371945932507515,
        "idea_test_rmse": 0.2369789406657219,
        "order_sensitivity": 0.36837229039520025,
        "system_gap": 0.3969488888978958
      },
      "confirmed": false
    },
    "notes": "Baseline and idea share the edge encoder and readout head; only additive aggregation versus ordered matrix multiplication differs."
  },
  "system_verdict": "worked",
  "practical_verdict": "helps",
  "mechanism_ok": 0,
  "system_judged": true
}