1{
  2  "bench_version": 1,
  3  "track": "sequence",
  4  "model": "transformer_tiny",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "best_cfg": {
  9      "lr": 0.0015
 10    },
 11    "sweep": [
 12      {
 13        "cfg": {
 14          "lr": 0.0007
 15        },
 16        "mean": 0.18085692822933197
 17      },
 18      {
 19        "cfg": {
 20          "lr": 0.001
 21        },
 22        "mean": 0.17446092143654823
 23      },
 24      {
 25        "cfg": {
 26          "lr": 0.0015
 27        },
 28        "mean": 0.17426595836877823
 29      },
 30      {
 31        "cfg": {
 32          "lr": 0.002
 33        },
 34        "mean": 0.1786409355700016
 35      },
 36      {
 37        "cfg": {
 38          "lr": 0.003
 39        },
 40        "mean": 0.18349025771021843
 41      }
 42    ],
 43    "full": {
 44      "mean": 0.16209838539361954,
 45      "std": 0.024681871508716122,
 46      "per_seed": [
 47        0.1827893853187561,
 48        0.13988947868347168,
 49        0.18467898666858673,
 50        0.1897059828042984,
 51        0.11963248252868652,
 52        0.15095271170139313,
 53        0.18380902707576752,
 54        0.14532902836799622
 55      ],
 56      "n": 8
 57    }
 58  },
 59  "idea": {
 60    "mean": 0.18732197023928165,
 61    "std": 0.025151544057018065,
 62    "per_seed": [
 63      0.19494226574897766,
 64      0.15557610988616943,
 65      0.19609080255031586,
 66      0.2065867781639099,
 67      0.1564088761806488,
 68      0.16889484226703644,
 69      0.23473307490348816,
 70      0.18534301221370697
 71    ],
 72    "n": 8
 73  },
 74  "comparison": {
 75    "delta_mean": 0.025223584845662117,
 76    "idea_wins": 0,
 77    "n_pairs": 8,
 78    "per_seed_diffs": [
 79      0.012152880430221558,
 80      0.015686631202697754,
 81      0.011411815881729126,
 82      0.01688079535961151,
 83      0.03677639365196228,
 84      0.01794213056564331,
 85      0.05092404782772064,
 86      0.040013983845710754
 87    ],
 88    "p_value": 0.0081,
 89    "mde": 0.01259364706714858,
 90    "mde_rel_pct": 7.769137882877572,
 91    "verdict": "idea worse (significant)",
 92    "system_worked": false
 93  },
 94  "mechanism_signature": {
 95    "idea_lr_sweep": [
 96      {
 97        "lr": 0.001,
 98        "result": {
 99          "mean": 0.2357968259602785,
100          "std": 0.02821421648971268,
101          "per_seed": [
102            0.23772037029266357,
103            0.20389337837696075,
104            0.24708855152130127,
105            0.242384672164917,
106            0.21312211453914642,
107            0.2053222507238388,
108            0.2967439293861389,
109            0.24009934067726135
110          ],
111          "n": 8
112        }
113      },
114      {
115        "lr": 0.0015,
116        "result": {
117          "mean": 0.20389264076948166,
118          "std": 0.024621166913138676,
119          "per_seed": [
120            0.21074053645133972,
121            0.1622573882341385,
122            0.21052612364292145,
123            0.21601423621177673,
124            0.1940460503101349,
125            0.18469412624835968,
126            0.2528582811355591,
127            0.20000438392162323
128          ],
129          "n": 8
130        }
131      },
132      {
133        "lr": 0.002,
134        "result": {
135          "mean": 0.18732197023928165,
136          "std": 0.025151544057018065,
137          "per_seed": [
138            0.19494226574897766,
139            0.15557610988616943,
140            0.19609080255031586,
141            0.2065867781639099,
142            0.1564088761806488,
143            0.16889484226703644,
144            0.23473307490348816,
145            0.18534301221370697
146          ],
147          "n": 8
148        }
149      }
150    ],
151    "selected_lr": 0.002,
152    "trained_model": true,
153    "predicted": {
154      "max_frobenius_ratio": "<=1",
155      "row_col_error": "0"
156    },
157    "observed": {
158      "max_frobenius_ratio": 0.7491575479507446,
159      "mean_frobenius_ratio": 0.748988151550293,
160      "max_row_error": 2.384185791015625e-07,
161      "max_col_error": 1.1920928955078125e-07
162    },
163    "confirmed": true,
164    "metric_on_signature_model": 0.19494226574897766
165  },
166  "custom_track": null
167}