{
  "bench_version": 1,
  "track": "sequence",
  "model": "transformer_tiny",
  "metric_direction": "lower is better",
  "n_seeds": 8,
  "baseline": {
    "best_cfg": {
      "lr": 0.0015
    },
    "sweep": [
      {
        "cfg": {
          "lr": 0.0007
        },
        "mean": 0.18085692822933197
      },
      {
        "cfg": {
          "lr": 0.001
        },
        "mean": 0.17446092143654823
      },
      {
        "cfg": {
          "lr": 0.0015
        },
        "mean": 0.17426595836877823
      },
      {
        "cfg": {
          "lr": 0.002
        },
        "mean": 0.1786409355700016
      },
      {
        "cfg": {
          "lr": 0.003
        },
        "mean": 0.18349025771021843
      }
    ],
    "full": {
      "mean": 0.16209838539361954,
      "std": 0.024681871508716122,
      "per_seed": [
        0.1827893853187561,
        0.13988947868347168,
        0.18467898666858673,
        0.1897059828042984,
        0.11963248252868652,
        0.15095271170139313,
        0.18380902707576752,
        0.14532902836799622
      ],
      "n": 8
    }
  },
  "idea": {
    "mean": 0.18732197023928165,
    "std": 0.025151544057018065,
    "per_seed": [
      0.19494226574897766,
      0.15557610988616943,
      0.19609080255031586,
      0.2065867781639099,
      0.1564088761806488,
      0.16889484226703644,
      0.23473307490348816,
      0.18534301221370697
    ],
    "n": 8
  },
  "comparison": {
    "delta_mean": 0.025223584845662117,
    "idea_wins": 0,
    "n_pairs": 8,
    "per_seed_diffs": [
      0.012152880430221558,
      0.015686631202697754,
      0.011411815881729126,
      0.01688079535961151,
      0.03677639365196228,
      0.01794213056564331,
      0.05092404782772064,
      0.040013983845710754
    ],
    "p_value": 0.0081,
    "mde": 0.01259364706714858,
    "mde_rel_pct": 7.769137882877572,
    "verdict": "idea worse (significant)",
    "system_worked": false
  },
  "mechanism_signature": {
    "idea_lr_sweep": [
      {
        "lr": 0.001,
        "result": {
          "mean": 0.2357968259602785,
          "std": 0.02821421648971268,
          "per_seed": [
            0.23772037029266357,
            0.20389337837696075,
            0.24708855152130127,
            0.242384672164917,
            0.21312211453914642,
            0.2053222507238388,
            0.2967439293861389,
            0.24009934067726135
          ],
          "n": 8
        }
      },
      {
        "lr": 0.0015,
        "result": {
          "mean": 0.20389264076948166,
          "std": 0.024621166913138676,
          "per_seed": [
            0.21074053645133972,
            0.1622573882341385,
            0.21052612364292145,
            0.21601423621177673,
            0.1940460503101349,
            0.18469412624835968,
            0.2528582811355591,
            0.20000438392162323
          ],
          "n": 8
        }
      },
      {
        "lr": 0.002,
        "result": {
          "mean": 0.18732197023928165,
          "std": 0.025151544057018065,
          "per_seed": [
            0.19494226574897766,
            0.15557610988616943,
            0.19609080255031586,
            0.2065867781639099,
            0.1564088761806488,
            0.16889484226703644,
            0.23473307490348816,
            0.18534301221370697
          ],
          "n": 8
        }
      }
    ],
    "selected_lr": 0.002,
    "trained_model": true,
    "predicted": {
      "max_frobenius_ratio": "<=1",
      "row_col_error": "0"
    },
    "observed": {
      "max_frobenius_ratio": 0.7491575479507446,
      "mean_frobenius_ratio": 0.748988151550293,
      "max_row_error": 2.384185791015625e-07,
      "max_col_error": 1.1920928955078125e-07
    },
    "confirmed": true,
    "metric_on_signature_model": 0.19494226574897766
  },
  "custom_track": null
}