1{
  2  "bench_version": 1,
  3  "track": "sequence",
  4  "model": "transformer_tiny",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "best_cfg": {
  9      "lr": 0.001
 10    },
 11    "sweep": [
 12      {
 13        "cfg": {
 14          "lr": 0.001
 15        },
 16        "mean": 0.9495677947998047
 17      },
 18      {
 19        "cfg": {
 20          "lr": 0.003
 21        },
 22        "mean": 1.5083084404468536
 23      },
 24      {
 25        "cfg": {
 26          "lr": 0.01
 27        },
 28        "mean": 3.8293468952178955
 29      }
 30    ],
 31    "full": {
 32      "mean": 1.006878174841404,
 33      "std": 0.1713716806448823,
 34      "per_seed": [
 35        0.9911782145500183,
 36        0.7554877996444702,
 37        1.2632131576538086,
 38        0.7883920073509216,
 39        1.1533313989639282,
 40        0.9415301084518433,
 41        1.1843085289001465,
 42        0.977584183216095
 43      ],
 44      "n": 8
 45    },
 46    "idea_grid": [
 47      {
 48        "cfg": {
 49          "lr": 0.001
 50        },
 51        "mean": 1.008343055844307
 52      },
 53      {
 54        "cfg": {
 55          "lr": 0.003
 56        },
 57        "mean": 1.4822083115577698
 58      },
 59      {
 60        "cfg": {
 61          "lr": 0.01
 62        },
 63        "mean": 4.900031566619873
 64      }
 65    ]
 66  },
 67  "idea": {
 68    "mean": 1.008343055844307,
 69    "std": 0.1720253375372032,
 70    "per_seed": [
 71      0.9909452795982361,
 72      0.7562634944915771,
 73      1.2659306526184082,
 74      0.7895816564559937,
 75      1.153704047203064,
 76      0.9436419606208801,
 77      1.1885106563568115,
 78      0.9781666994094849
 79    ],
 80    "n": 8
 81  },
 82  "comparison": {
 83    "delta_mean": 0.0014648810029029846,
 84    "idea_wins": 1,
 85    "n_pairs": 8,
 86    "per_seed_diffs": [
 87      -0.00023293495178222656,
 88      0.0007756948471069336,
 89      0.0027174949645996094,
 90      0.0011896491050720215,
 91      0.0003726482391357422,
 92      0.0021118521690368652,
 93      0.004202127456665039,
 94      0.0005825161933898926
 95    ],
 96    "p_value": 0.01485,
 97    "mde": 0.0012193019578819895,
 98    "mde_rel_pct": 0.12109726760876957,
 99    "verdict": "idea worse (significant)",
100    "system_worked": false
101  },
102  "mechanism_signature": {
103    "mean_hidden": 0.49465957656502724,
104    "saturated_fraction": 0.0,
105    "projection_alignment": 0.020752153369809595,
106    "per_seed": [
107      {
108        "mean_hidden": 0.4961187243461609,
109        "saturated_fraction": 0.0,
110        "projection_alignment": 0.014636302551895108,
111        "device": "cuda"
112      },
113      {
114        "mean_hidden": 0.5012550354003906,
115        "saturated_fraction": 0.0,
116        "projection_alignment": 0.022799921693492292,
117        "device": "cuda"
118      },
119      {
120        "mean_hidden": 0.4911428391933441,
121        "saturated_fraction": 0.0,
122        "projection_alignment": 0.02356167507297876,
123        "device": "cuda"
124      },
125      {
126        "mean_hidden": 0.4819362759590149,
127        "saturated_fraction": 0.0,
128        "projection_alignment": 0.02155706124715023,
129        "device": "cuda"
130      },
131      {
132        "mean_hidden": 0.5072823166847229,
133        "saturated_fraction": 0.0,
134        "projection_alignment": 0.022357459330866497,
135        "device": "cuda"
136      },
137      {
138        "mean_hidden": 0.4996195137500763,
139        "saturated_fraction": 0.0,
140        "projection_alignment": 0.025012486868765855,
141        "device": "cuda"
142      },
143      {
144        "mean_hidden": 0.4985620379447937,
145        "saturated_fraction": 0.0,
146        "projection_alignment": 0.0183842962110037,
147        "device": "cuda"
148      },
149      {
150        "mean_hidden": 0.4813598692417145,
151        "saturated_fraction": 0.0,
152        "projection_alignment": 0.017708023982324326,
153        "device": "cuda"
154      }
155    ],
156    "predicted_projection_alignment_positive": true,
157    "confirmed": true
158  },
159  "protocol_notes": {
160    "architecture_match": "identical sigmoid-input transformer; only inp training rule differs",
161    "track_justification": "sequence forecast contains ordered multi-token temporal windows",
162    "budget": {
163      "n_train": 400,
164      "n_test": 400,
165      "epochs": 5,
166      "batch": 128,
167      "lr_union": [
168        0.001,
169        0.003,
170        0.01
171      ]
172    },
173    "custom_track": null
174  }
175}