{
  "bench_version": 1,
  "track": "sequence",
  "model": "transformer_tiny",
  "metric_direction": "lower is better",
  "n_seeds": 8,
  "baseline": {
    "best_cfg": {
      "lr": 0.001,
      "temperature": 1.0
    },
    "sweep": [
      {
        "cfg": {
          "lr": 0.001,
          "temperature": 0.7
        },
        "mean": 0.6538640260696411
      },
      {
        "cfg": {
          "lr": 0.001,
          "temperature": 1.0
        },
        "mean": 0.6523745656013489
      },
      {
        "cfg": {
          "lr": 0.003,
          "temperature": 0.7
        },
        "mean": 0.6653543412685394
      },
      {
        "cfg": {
          "lr": 0.003,
          "temperature": 1.0
        },
        "mean": 0.6606301814317703
      },
      {
        "cfg": {
          "lr": 0.006,
          "temperature": 0.7
        },
        "mean": 0.7325459271669388
      },
      {
        "cfg": {
          "lr": 0.006,
          "temperature": 1.0
        },
        "mean": 0.7683279514312744
      }
    ],
    "full": {
      "mean": 0.6796555295586586,
      "std": 0.05923041132805598,
      "per_seed": [
        0.7301222085952759,
        0.6001890897750854,
        0.6041663885116577,
        0.6750205755233765,
        0.7771981358528137,
        0.6423909664154053,
        0.7304211258888245,
        0.6777357459068298
      ],
      "n": 8
    }
  },
  "idea": {
    "mean": 0.647096149623394,
    "std": 0.07594962062926218,
    "per_seed": [
      0.774990975856781,
      0.5947428941726685,
      0.5252495408058167,
      0.6463086605072021,
      0.7060304880142212,
      0.5773214101791382,
      0.7090907096862793,
      0.6430345177650452
    ],
    "n": 8
  },
  "comparison": {
    "delta_mean": -0.03255937993526459,
    "idea_wins": 7,
    "n_pairs": 8,
    "per_seed_diffs": [
      0.04486876726150513,
      -0.005446195602416992,
      -0.07891684770584106,
      -0.028711915016174316,
      -0.07116764783859253,
      -0.06506955623626709,
      -0.021330416202545166,
      -0.03470122814178467
    ],
    "p_value": 0.0673,
    "mde": 0.033997422326021196,
    "mde_rel_pct": 5.002154892802501,
    "verdict": "no significant win",
    "system_worked": false
  },
  "mechanism_signature": {
    "trained_model": true,
    "M": 32,
    "observed_rmse_by_M": [
      {
        "M": 8,
        "rmse": 0.4599074721336365
      },
      {
        "M": 16,
        "rmse": 0.35632288455963135
      },
      {
        "M": 32,
        "rmse": 0.22849832475185394
      }
    ],
    "observed_slope": -0.5045800246521701,
    "predicted_slope": -0.5,
    "signed_rmse": 0.22849832475185394,
    "unsigned_rmse": 0.38780272006988525,
    "confirmed": true
  },
  "idea_sweep": [
    {
      "cfg": {
        "lr": 0.001,
        "m": 32
      },
      "result": {
        "mean": 0.7032602652907372,
        "std": 0.11299721951432162,
        "per_seed": [
          0.7925687432289124,
          0.6065788269042969,
          0.5842217206954956,
          0.6156045198440552,
          0.8893585801124573,
          0.5813699960708618,
          0.74463951587677,
          0.8117402195930481
        ],
        "n": 8
      }
    },
    {
      "cfg": {
        "lr": 0.003,
        "m": 32
      },
      "result": {
        "mean": 0.647096149623394,
        "std": 0.07594962062926218,
        "per_seed": [
          0.774990975856781,
          0.5947428941726685,
          0.5252495408058167,
          0.6463086605072021,
          0.7060304880142212,
          0.5773214101791382,
          0.7090907096862793,
          0.6430345177650452
        ],
        "n": 8
      }
    },
    {
      "cfg": {
        "lr": 0.006,
        "m": 32
      },
      "result": {
        "mean": 0.9036180377006531,
        "std": 0.0674238212671651,
        "per_seed": [
          0.9230982661247253,
          0.871337890625,
          0.8432034254074097,
          0.7719256281852722,
          0.9592460989952087,
          0.9292699694633484,
          1.0012027025222778,
          0.9296603202819824
        ],
        "n": 8
      }
    }
  ],
  "sanity_check": {
    "rows": [
      {
        "M": 16,
        "signed_rmse": 0.3650483780576059,
        "positive_rmse": 0.6758467186361339
      },
      {
        "M": 32,
        "signed_rmse": 0.20704154016476478,
        "positive_rmse": 0.6506880673538492
      },
      {
        "M": 64,
        "signed_rmse": 0.16207412164394386,
        "positive_rmse": 0.6397375154354732
      },
      {
        "M": 128,
        "signed_rmse": 0.12449882232224384,
        "positive_rmse": 0.6258620222659124
      },
      {
        "M": 256,
        "signed_rmse": 0.08409067839559449,
        "positive_rmse": 0.6281025092775255
      }
    ],
    "observed_slope": -0.49699279208004493,
    "predicted_slope": -0.5,
    "positive_predicted_floor": 0.626260339536197,
    "passed": true
  },
  "protocol": {
    "seeds": [
      0,
      1,
      2,
      3,
      4,
      5,
      6,
      7
    ],
    "n_train": 400,
    "n_test": 200,
    "epochs": 6,
    "baseline_grid": [
      {
        "lr": 0.001,
        "temperature": 0.7
      },
      {
        "lr": 0.001,
        "temperature": 1.0
      },
      {
        "lr": 0.003,
        "temperature": 0.7
      },
      {
        "lr": 0.003,
        "temperature": 1.0
      },
      {
        "lr": 0.006,
        "temperature": 0.7
      },
      {
        "lr": 0.006,
        "temperature": 1.0
      }
    ],
    "idea_grid": [
      {
        "lr": 0.001,
        "m": 32
      },
      {
        "lr": 0.003,
        "m": 32
      },
      {
        "lr": 0.006,
        "m": 32
      }
    ]
  }
}