{
  "bench_version": 1,
  "track": "masked_multitoken_modes",
  "model": "transformer_tiny",
  "metric_direction": "lower is better",
  "n_seeds": 8,
  "baseline": {
    "best_cfg": {
      "lr": 0.002,
      "epochs": 14,
      "hide_count": 1
    },
    "sweep": [
      {
        "cfg": {
          "lr": 0.002,
          "epochs": 14,
          "hide_count": 1
        },
        "mean": 0.1428266018629074
      },
      {
        "cfg": {
          "lr": 0.004,
          "epochs": 14,
          "hide_count": 1
        },
        "mean": 0.20178561471402645
      },
      {
        "cfg": {
          "lr": 0.008,
          "epochs": 14,
          "hide_count": 1
        },
        "mean": 0.14775850623846054
      }
    ],
    "full": {
      "mean": 0.14373092539608479,
      "std": 0.02385596096955299,
      "per_seed": [
        0.1670890897512436,
        0.12829364836215973,
        0.13317705690860748,
        0.14274661242961884,
        0.16149330139160156,
        0.09596626460552216,
        0.14481675624847412,
        0.1762646734714508
      ],
      "n": 8
    }
  },
  "idea": {
    "mean": 0.14993123710155487,
    "std": 0.038141062047303144,
    "per_seed": [
      0.1759413331747055,
      0.13947612047195435,
      0.12913769483566284,
      0.14234721660614014,
      0.149277925491333,
      0.09132890403270721,
      0.13944129645824432,
      0.2324994057416916
    ],
    "n": 8,
    "best_cfg": {
      "lr": 0.004,
      "epochs": 14,
      "lam": 0.1,
      "rho": 0.01
    }
  },
  "comparison": {
    "delta_mean": 0.006200311705470085,
    "idea_wins": 5,
    "n_pairs": 8,
    "per_seed_diffs": [
      0.008852243423461914,
      0.011182472109794617,
      -0.004039362072944641,
      -0.00039939582347869873,
      -0.012215375900268555,
      -0.004637360572814941,
      -0.005375459790229797,
      0.056234732270240784
    ],
    "p_value": 0.63155,
    "mde": 0.018084067093522574,
    "mde_rel_pct": 12.581890114244809,
    "verdict": "no significant win",
    "system_worked": false
  },
  "mechanism_signature": {
    "schedule_check": [
      {
        "lambda": 0,
        "rho": 0,
        "pred_pi1": 0,
        "obs_pi1": 0.0,
        "pred_full": 0,
        "obs_full": 0.0,
        "v_min": 7
      },
      {
        "lambda": 0.1,
        "rho": 0,
        "pred_pi1": 0.1,
        "obs_pi1": 0.10140000283718109,
        "pred_full": 0,
        "obs_full": 0.0,
        "v_min": 0
      },
      {
        "lambda": 0.1,
        "rho": 0.01,
        "pred_pi1": 0.11,
        "obs_pi1": 0.11155000329017639,
        "pred_full": 0.01,
        "obs_full": 0.009879999794065952,
        "v_min": 0
      },
      {
        "lambda": 0.25,
        "rho": 0.01,
        "pred_pi1": 0.26,
        "obs_pi1": 0.26243001222610474,
        "pred_full": 0.01,
        "obs_full": 0.009879999794065952,
        "v_min": 0
      }
    ],
    "signature": {
      "quantity": "empty-context mode probability",
      "target_true_weight": 0.8,
      "predicted_baseline": 0.8209748044610023,
      "predicted_idea": 0.839116208255291,
      "observed_baseline": 0.8209748044610023,
      "observed_idea": 0.839116208255291,
      "predicted_improvement": -0.018141403794288635,
      "confirmed": false
    },
    "custom_track": {
      "name": "masked_multitoken_modes",
      "file": "custom_masked_track.py",
      "domain": "sequence"
    }
  },
  "idea_sweep": [
    {
      "cfg": {
        "lr": 0.002,
        "epochs": 14,
        "lam": 0.1,
        "rho": 0.0
      },
      "mean": 0.16454514022916555,
      "per_seed": [
        0.16334068775177002,
        0.22535602748394012,
        0.12713226675987244,
        0.18480685353279114,
        0.19457480311393738,
        0.10892865806818008,
        0.14529073238372803,
        0.16693109273910522
      ],
      "mode_probs": [
        0.7183287739753723,
        0.8767662644386292,
        0.9214068055152893,
        0.8836508393287659,
        0.8848533034324646,
        0.7462123036384583,
        0.780626654624939,
        0.8711745738983154
      ]
    },
    {
      "cfg": {
        "lr": 0.004,
        "epochs": 14,
        "lam": 0.1,
        "rho": 0.01
      },
      "mean": 0.14993123710155487,
      "per_seed": [
        0.1759413331747055,
        0.13947612047195435,
        0.12913769483566284,
        0.14234721660614014,
        0.149277925491333,
        0.09132890403270721,
        0.13944129645824432,
        0.2324994057416916
      ],
      "mode_probs": [
        0.9315788745880127,
        0.8918474316596985,
        0.8515940308570862,
        0.9011403918266296,
        0.7730483412742615,
        0.7809845209121704,
        0.7969619035720825,
        0.7857741713523865
      ]
    },
    {
      "cfg": {
        "lr": 0.008,
        "epochs": 14,
        "lam": 0.25,
        "rho": 0.01
      },
      "mean": 0.18514209240674973,
      "per_seed": [
        0.18203185498714447,
        0.16977907717227936,
        0.13093167543411255,
        0.2541387677192688,
        0.21239057183265686,
        0.16110031306743622,
        0.14696472883224487,
        0.22379975020885468
      ],
      "mode_probs": [
        0.7211427092552185,
        0.8639533519744873,
        0.8087149262428284,
        0.8416880965232849,
        0.8448388576507568,
        0.8393517136573792,
        0.7988701462745667,
        0.8555705547332764
      ]
    }
  ]
}