1{
  2  "bench_version": 1,
  3  "track": "sequence",
  4  "model": "transformer_tiny",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "best_cfg": {
  9      "lr": 0.001
 10    },
 11    "sweep": [
 12      {
 13        "cfg": {
 14          "lr": 0.001
 15        },
 16        "mean": 0.47025539726018906
 17      },
 18      {
 19        "cfg": {
 20          "lr": 0.003
 21        },
 22        "mean": 0.5620733425021172
 23      },
 24      {
 25        "cfg": {
 26          "lr": 0.006
 27        },
 28        "mean": 0.7826127856969833
 29      }
 30    ],
 31    "full": {
 32      "mean": 0.482548501342535,
 33      "std": 0.03354559876849172,
 34      "per_seed": [
 35        0.4842072129249573,
 36        0.42341384291648865,
 37        0.4566255807876587,
 38        0.5167749524116516,
 39        0.49487417936325073,
 40        0.4824545681476593,
 41        0.5386028289794922,
 42        0.4634348452091217
 43      ],
 44      "n": 8
 45    }
 46  },
 47  "idea": {
 48    "mean": 0.4949374906718731,
 49    "std": 0.053264005967373906,
 50    "per_seed": [
 51      0.48880892992019653,
 52      0.4039829969406128,
 53      0.4540945589542389,
 54      0.5083529353141785,
 55      0.5644499659538269,
 56      0.5458484292030334,
 57      0.5475071668624878,
 58      0.4464549422264099
 59    ],
 60    "n": 8
 61  },
 62  "comparison": {
 63    "delta_mean": 0.012388989329338074,
 64    "idea_wins": 4,
 65    "n_pairs": 8,
 66    "per_seed_diffs": [
 67      0.004601716995239258,
 68      -0.019430845975875854,
 69      -0.0025310218334198,
 70      -0.008422017097473145,
 71      0.06957578659057617,
 72      0.06339386105537415,
 73      0.008904337882995605,
 74      -0.016979902982711792
 75    ],
 76    "p_value": 0.42645,
 77    "mde": 0.02909483793289158,
 78    "mde_rel_pct": 6.029412142394932,
 79    "verdict": "no significant win",
 80    "system_worked": false
 81  },
 82  "track_match": "sequence-level correlated multi-token forecast",
 83  "idea_best_cfg": {
 84    "lr": 0.001
 85  },
 86  "idea_grid": [
 87    {
 88      "cfg": {
 89        "lr": 0.001
 90      },
 91      "full": {
 92        "mean": 0.4949374906718731,
 93        "std": 0.053264005967373906,
 94        "per_seed": [
 95          0.48880892992019653,
 96          0.4039829969406128,
 97          0.4540945589542389,
 98          0.5083529353141785,
 99          0.5644499659538269,
100          0.5458484292030334,
101          0.5475071668624878,
102          0.4464549422264099
103        ],
104        "n": 8
105      }
106    },
107    {
108      "cfg": {
109        "lr": 0.003
110      },
111      "full": {
112        "mean": 0.6153423972427845,
113        "std": 0.08454027758325999,
114        "per_seed": [
115          0.5847313404083252,
116          0.42449966073036194,
117          0.6188046336174011,
118          0.6627813577651978,
119          0.7065386772155762,
120          0.6074619889259338,
121          0.7107876539230347,
122          0.6071338653564453
123        ],
124        "n": 8
125      }
126    },
127    {
128      "cfg": {
129        "lr": 0.006
130      },
131      "full": {
132        "mean": 0.8792074918746948,
133        "std": 0.3064714056914715,
134        "per_seed": [
135          0.6058098673820496,
136          0.64018714427948,
137          0.6481318473815918,
138          0.9747511148452759,
139          1.0847872495651245,
140          0.8413496017456055,
141          1.5654103755950928,
142          0.6732327342033386
143        ],
144        "n": 8
145      }
146    }
147  ],
148  "epochs": 10,
149  "n_train": 400,
150  "n_test": 200,
151  "mechanism_signature": {
152    "mean_baseline_output_noise_ratio": 0.17996257916092873,
153    "mean_idea_output_noise_ratio": 0.16205820441246033,
154    "predicted": "idea should attenuate feature perturbation",
155    "confirmed": true,
156    "per_seed": [
157      {
158        "baseline_output_noise_ratio": 0.09764544665813446,
159        "idea_output_noise_ratio": 0.1503368318080902,
160        "baseline_metric": 0.4308149516582489,
161        "idea_metric": 0.5847313404083252
162      },
163      {
164        "baseline_output_noise_ratio": 0.14380529522895813,
165        "idea_output_noise_ratio": 0.1423303335905075,
166        "baseline_metric": 0.4464939832687378,
167        "idea_metric": 0.42449966073036194
168      },
169      {
170        "baseline_output_noise_ratio": 0.2634066641330719,
171        "idea_output_noise_ratio": 0.2061924785375595,
172        "baseline_metric": 0.6809554696083069,
173        "idea_metric": 0.6188046336174011
174      },
175      {
176        "baseline_output_noise_ratio": 0.20416639745235443,
177        "idea_output_noise_ratio": 0.15186598896980286,
178        "baseline_metric": 0.690028965473175,
179        "idea_metric": 0.6627813577651978
180      },
181      {
182        "baseline_output_noise_ratio": 0.13947272300720215,
183        "idea_output_noise_ratio": 0.1979530155658722,
184        "baseline_metric": 0.7755821347236633,
185        "idea_metric": 0.7065386772155762
186      },
187      {
188        "baseline_output_noise_ratio": 0.21130000054836273,
189        "idea_output_noise_ratio": 0.1443224996328354,
190        "baseline_metric": 0.7781144380569458,
191        "idea_metric": 0.6074619889259338
192      },
193      {
194        "baseline_output_noise_ratio": 0.2646835446357727,
195        "idea_output_noise_ratio": 0.11280205845832825,
196        "baseline_metric": 0.655863881111145,
197        "idea_metric": 0.7107876539230347
198      },
199      {
200        "baseline_output_noise_ratio": 0.1152205616235733,
201        "idea_output_noise_ratio": 0.1906624287366867,
202        "baseline_metric": 0.6570667028427124,
203        "idea_metric": 0.6071338653564453
204      }
205    ]
206  }
207}