1{
  2  "bench_version": 1,
  3  "track": "sequence",
  4  "model": "transformer_tiny",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "best_cfg": {
  9      "lr": 0.0015
 10    },
 11    "sweep": [
 12      {
 13        "cfg": {
 14          "lr": 0.0015
 15        },
 16        "mean": 0.6402924656867981
 17      },
 18      {
 19        "cfg": {
 20          "lr": 0.003
 21        },
 22        "mean": 0.7056357711553574
 23      },
 24      {
 25        "cfg": {
 26          "lr": 0.006
 27        },
 28        "mean": 0.9283334761857986
 29      }
 30    ],
 31    "full": {
 32      "mean": 0.6229188442230225,
 33      "std": 0.06931051155059516,
 34      "per_seed": [
 35        0.7044264674186707,
 36        0.5437753200531006,
 37        0.5856346487998962,
 38        0.7273334264755249,
 39        0.5151234269142151,
 40        0.6567925214767456,
 41        0.6100528240203857,
 42        0.6402121186256409
 43      ],
 44      "n": 8
 45    }
 46  },
 47  "idea": {
 48    "best_cfg": {
 49      "lr": 0.0015,
 50      "beta": 0.2,
 51      "eps": 0.15
 52    },
 53    "sweep": [
 54      {
 55        "cfg": {
 56          "lr": 0.0015,
 57          "beta": 0.2,
 58          "eps": 0.15
 59        },
 60        "mean": 0.4291221722960472
 61      },
 62      {
 63        "cfg": {
 64          "lr": 0.003,
 65          "beta": 0.3,
 66          "eps": 0.15
 67        },
 68        "mean": 0.4798029735684395
 69      },
 70      {
 71        "cfg": {
 72          "lr": 0.006,
 73          "beta": 0.4,
 74          "eps": 0.2
 75        },
 76        "mean": 0.6943782418966293
 77      }
 78    ],
 79    "per_seed": [
 80      0.49007120728492737,
 81      0.31841030716896057,
 82      0.4649355113506317,
 83      0.4430716633796692,
 84      0.3909056484699249,
 85      0.4021309018135071,
 86      0.49423667788505554,
 87      0.3951165974140167
 88    ]
 89  },
 90  "comparison": {
 91    "delta_mean": -0.19805902987718582,
 92    "idea_wins": 8,
 93    "n_pairs": 8,
 94    "per_seed_diffs": [
 95      -0.2143552601337433,
 96      -0.22536501288414001,
 97      -0.12069913744926453,
 98      -0.2842617630958557,
 99      -0.12421777844429016,
100      -0.2546616196632385,
101      -0.1158161461353302,
102      -0.24509552121162415
103    ],
104    "p_value": 0.0081,
105    "mde": 0.05658672011309537,
106    "mde_rel_pct": 9.084123981459733,
107    "verdict": "idea better (significant)",
108    "system_worked": true
109  },
110  "mechanism_signature": {
111    "prediction": "relational compatibility reduces incompatible-token attention",
112    "predicted": {
113      "partial_row_upper_bound": 0.03125,
114      "beta": 0.2
115    },
116    "observed": {
117      "baseline_test_mse_mean": 0.6229188442230225,
118      "idea_test_mse_mean": 0.42485981434583664
119    },
120    "confirmed": false
121  },
122  "protocol_notes": {
123    "track_reason": "The sequence track is multi-token temporal forecasting with transformer attention, directly matching the proposed mechanism.",
124    "reduced_budget": "600 train/300 test and 5 epochs due to PGW O(L^4) cost; paired seeds remain 8; baseline sweep uses the same lr union.",
125    "elapsed_sec": 37.595829248428345
126  }
127}