{
  "bench_version": 1,
  "track": "sequence",
  "model": "transformer_tiny",
  "metric_direction": "lower is better",
  "n_seeds": 8,
  "baseline": {
    "best_cfg": {
      "lr": 0.001
    },
    "sweep": [
      {
        "cfg": {
          "lr": 0.001
        },
        "mean": 0.47025539726018906
      },
      {
        "cfg": {
          "lr": 0.003
        },
        "mean": 0.5620733425021172
      },
      {
        "cfg": {
          "lr": 0.006
        },
        "mean": 0.7826127856969833
      }
    ],
    "full": {
      "mean": 0.482548501342535,
      "std": 0.03354559876849172,
      "per_seed": [
        0.4842072129249573,
        0.42341384291648865,
        0.4566255807876587,
        0.5167749524116516,
        0.49487417936325073,
        0.4824545681476593,
        0.5386028289794922,
        0.4634348452091217
      ],
      "n": 8
    }
  },
  "idea": {
    "mean": 0.4949374906718731,
    "std": 0.053264005967373906,
    "per_seed": [
      0.48880892992019653,
      0.4039829969406128,
      0.4540945589542389,
      0.5083529353141785,
      0.5644499659538269,
      0.5458484292030334,
      0.5475071668624878,
      0.4464549422264099
    ],
    "n": 8
  },
  "comparison": {
    "delta_mean": 0.012388989329338074,
    "idea_wins": 4,
    "n_pairs": 8,
    "per_seed_diffs": [
      0.004601716995239258,
      -0.019430845975875854,
      -0.0025310218334198,
      -0.008422017097473145,
      0.06957578659057617,
      0.06339386105537415,
      0.008904337882995605,
      -0.016979902982711792
    ],
    "p_value": 0.42645,
    "mde": 0.02909483793289158,
    "mde_rel_pct": 6.029412142394932,
    "verdict": "no significant win",
    "system_worked": false
  },
  "track_match": "sequence-level correlated multi-token forecast",
  "idea_best_cfg": {
    "lr": 0.001
  },
  "idea_grid": [
    {
      "cfg": {
        "lr": 0.001
      },
      "full": {
        "mean": 0.4949374906718731,
        "std": 0.053264005967373906,
        "per_seed": [
          0.48880892992019653,
          0.4039829969406128,
          0.4540945589542389,
          0.5083529353141785,
          0.5644499659538269,
          0.5458484292030334,
          0.5475071668624878,
          0.4464549422264099
        ],
        "n": 8
      }
    },
    {
      "cfg": {
        "lr": 0.003
      },
      "full": {
        "mean": 0.6153423972427845,
        "std": 0.08454027758325999,
        "per_seed": [
          0.5847313404083252,
          0.42449966073036194,
          0.6188046336174011,
          0.6627813577651978,
          0.7065386772155762,
          0.6074619889259338,
          0.7107876539230347,
          0.6071338653564453
        ],
        "n": 8
      }
    },
    {
      "cfg": {
        "lr": 0.006
      },
      "full": {
        "mean": 0.8792074918746948,
        "std": 0.3064714056914715,
        "per_seed": [
          0.6058098673820496,
          0.64018714427948,
          0.6481318473815918,
          0.9747511148452759,
          1.0847872495651245,
          0.8413496017456055,
          1.5654103755950928,
          0.6732327342033386
        ],
        "n": 8
      }
    }
  ],
  "epochs": 10,
  "n_train": 400,
  "n_test": 200,
  "mechanism_signature": {
    "mean_baseline_output_noise_ratio": 0.17996257916092873,
    "mean_idea_output_noise_ratio": 0.16205820441246033,
    "predicted": "idea should attenuate feature perturbation",
    "confirmed": true,
    "per_seed": [
      {
        "baseline_output_noise_ratio": 0.09764544665813446,
        "idea_output_noise_ratio": 0.1503368318080902,
        "baseline_metric": 0.4308149516582489,
        "idea_metric": 0.5847313404083252
      },
      {
        "baseline_output_noise_ratio": 0.14380529522895813,
        "idea_output_noise_ratio": 0.1423303335905075,
        "baseline_metric": 0.4464939832687378,
        "idea_metric": 0.42449966073036194
      },
      {
        "baseline_output_noise_ratio": 0.2634066641330719,
        "idea_output_noise_ratio": 0.2061924785375595,
        "baseline_metric": 0.6809554696083069,
        "idea_metric": 0.6188046336174011
      },
      {
        "baseline_output_noise_ratio": 0.20416639745235443,
        "idea_output_noise_ratio": 0.15186598896980286,
        "baseline_metric": 0.690028965473175,
        "idea_metric": 0.6627813577651978
      },
      {
        "baseline_output_noise_ratio": 0.13947272300720215,
        "idea_output_noise_ratio": 0.1979530155658722,
        "baseline_metric": 0.7755821347236633,
        "idea_metric": 0.7065386772155762
      },
      {
        "baseline_output_noise_ratio": 0.21130000054836273,
        "idea_output_noise_ratio": 0.1443224996328354,
        "baseline_metric": 0.7781144380569458,
        "idea_metric": 0.6074619889259338
      },
      {
        "baseline_output_noise_ratio": 0.2646835446357727,
        "idea_output_noise_ratio": 0.11280205845832825,
        "baseline_metric": 0.655863881111145,
        "idea_metric": 0.7107876539230347
      },
      {
        "baseline_output_noise_ratio": 0.1152205616235733,
        "idea_output_noise_ratio": 0.1906624287366867,
        "baseline_metric": 0.6570667028427124,
        "idea_metric": 0.6071338653564453
      }
    ]
  }
}