{
  "bench_version": 1,
  "track": "sequence",
  "model": "transformer_tiny",
  "metric_direction": "lower is better",
  "n_seeds": 8,
  "baseline": {
    "best_cfg": {
      "lr": 0.001,
      "tau": 2.0
    },
    "sweep": [
      {
        "cfg": {
          "lr": 0.001,
          "tau": 0.5
        },
        "mean": 0.5767685323953629
      },
      {
        "cfg": {
          "lr": 0.001,
          "tau": 1.0
        },
        "mean": 0.5731862112879753
      },
      {
        "cfg": {
          "lr": 0.001,
          "tau": 2.0
        },
        "mean": 0.5661996975541115
      },
      {
        "cfg": {
          "lr": 0.003,
          "tau": 0.5
        },
        "mean": 0.7547104060649872
      },
      {
        "cfg": {
          "lr": 0.003,
          "tau": 1.0
        },
        "mean": 0.6658876091241837
      },
      {
        "cfg": {
          "lr": 0.003,
          "tau": 2.0
        },
        "mean": 0.7227341532707214
      },
      {
        "cfg": {
          "lr": 0.01,
          "tau": 0.5
        },
        "mean": 1.189420223236084
      },
      {
        "cfg": {
          "lr": 0.01,
          "tau": 1.0
        },
        "mean": 1.965364009141922
      },
      {
        "cfg": {
          "lr": 0.01,
          "tau": 2.0
        },
        "mean": 1.9628772139549255
      }
    ],
    "full": {
      "mean": 0.5629024468362331,
      "std": 0.0610525769421287,
      "per_seed": [
        0.639130711555481,
        0.47835078835487366,
        0.5805663466453552,
        0.5667509436607361,
        0.6444448828697205,
        0.4818141460418701,
        0.5961759686470032,
        0.5159857869148254
      ],
      "n": 8
    }
  },
  "idea": {
    "mean": 0.5631052926182747,
    "std": 0.05872795158375719,
    "per_seed": [
      0.6507832407951355,
      0.4800701141357422,
      0.6299005746841431,
      0.5685753226280212,
      0.5922347903251648,
      0.5042065382003784,
      0.5788382887840271,
      0.5002334713935852
    ],
    "n": 8
  },
  "comparison": {
    "delta_mean": 0.00020284578204154968,
    "idea_wins": 3,
    "n_pairs": 8,
    "per_seed_diffs": [
      0.011652529239654541,
      0.0017193257808685303,
      0.04933422803878784,
      0.0018243789672851562,
      -0.052210092544555664,
      0.0223923921585083,
      -0.017337679862976074,
      -0.015752315521240234
    ],
    "p_value": 0.9927,
    "mde": 0.025187222915370296,
    "mde_rel_pct": 4.47452716841682,
    "verdict": "no measurable effect",
    "system_worked": false
  },
  "mechanism_signature": {
    "prediction": "role-selective exact queries should be more concentrated than mismatched queries",
    "predicted_exact_minus_mismatched_mass": 0.2,
    "observed_exact_role_mass": 0.03502586856484413,
    "observed_mismatched_role_mass": 0.03239620546810329,
    "observed_gap": 0.002629663096740842,
    "confirmed": false,
    "note": "probability mass is measured from attention of trained role-filler models on held-out sequence windows"
  },
  "idea_sweep": [
    {
      "cfg": {
        "lr": 0.001,
        "tau": 2.0
      },
      "result": {
        "mean": 0.5641730912029743,
        "std": 0.0628611757188025,
        "per_seed": [
          0.6497918367385864,
          0.4751982092857361,
          0.6231837868690491,
          0.5758178234100342,
          0.5922346711158752,
          0.49082884192466736,
          0.6094589233398438,
          0.49687063694000244
        ],
        "n": 8
      }
    },
    {
      "cfg": {
        "lr": 0.001,
        "tau": 1.0
      },
      "result": {
        "mean": 0.5631052926182747,
        "std": 0.05872795158375719,
        "per_seed": [
          0.6507832407951355,
          0.4800701141357422,
          0.6299005746841431,
          0.5685753226280212,
          0.5922347903251648,
          0.5042065382003784,
          0.5788382887840271,
          0.5002334713935852
        ],
        "n": 8
      }
    },
    {
      "cfg": {
        "lr": 0.01,
        "tau": 1.0
      },
      "result": {
        "mean": 1.587875448167324,
        "std": 0.6160067425650491,
        "per_seed": [
          1.209954857826233,
          2.5395073890686035,
          2.092766761779785,
          0.9216881990432739,
          0.9916467070579529,
          1.3980218172073364,
          2.4178340435028076,
          1.1315838098526
        ],
        "n": 8
      }
    }
  ]
}