1{
  2  "bench_version": 1,
  3  "track": "distribution_token_sequence",
  4  "model": "custom_poolnet",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "best_cfg": {
  9      "lr": 0.006,
 10      "epochs": 12,
 11      "delta": 0.5
 12    },
 13    "sweep": [
 14      {
 15        "cfg": {
 16          "lr": 0.001,
 17          "epochs": 12,
 18          "delta": 0.5
 19        },
 20        "mean": 0.11337076500058174
 21      },
 22      {
 23        "cfg": {
 24          "lr": 0.001,
 25          "epochs": 12,
 26          "delta": 0.8
 27        },
 28        "mean": 0.11337076500058174
 29      },
 30      {
 31        "cfg": {
 32          "lr": 0.001,
 33          "epochs": 12,
 34          "delta": 1.2
 35        },
 36        "mean": 0.11337076500058174
 37      },
 38      {
 39        "cfg": {
 40          "lr": 0.003,
 41          "epochs": 12,
 42          "delta": 0.5
 43        },
 44        "mean": 0.09198673628270626
 45      },
 46      {
 47        "cfg": {
 48          "lr": 0.003,
 49          "epochs": 12,
 50          "delta": 0.8
 51        },
 52        "mean": 0.09198673628270626
 53      },
 54      {
 55        "cfg": {
 56          "lr": 0.003,
 57          "epochs": 12,
 58          "delta": 1.2
 59        },
 60        "mean": 0.09198673628270626
 61      },
 62      {
 63        "cfg": {
 64          "lr": 0.006,
 65          "epochs": 12,
 66          "delta": 0.5
 67        },
 68        "mean": 0.06806101836264133
 69      },
 70      {
 71        "cfg": {
 72          "lr": 0.006,
 73          "epochs": 12,
 74          "delta": 0.8
 75        },
 76        "mean": 0.06806101836264133
 77      },
 78      {
 79        "cfg": {
 80          "lr": 0.006,
 81          "epochs": 12,
 82          "delta": 1.2
 83        },
 84        "mean": 0.06806101836264133
 85      }
 86    ],
 87    "full": {
 88      "mean": 0.06672146823257208,
 89      "std": 0.011285819903738186,
 90      "per_seed": [
 91        0.06364592909812927,
 92        0.0627693384885788,
 93        0.08224158734083176,
 94        0.06358721852302551,
 95        0.05995246767997742,
 96        0.08138634264469147,
 97        0.04605041444301605,
 98        0.07413844764232635
 99      ],
100      "n": 8
101    }
102  },
103  "idea": {
104    "mean": 0.018798072007484734,
105    "std": 0.0037972447648455363,
106    "per_seed": [
107      0.017789283767342567,
108      0.02162032574415207,
109      0.024604864418506622,
110      0.01876969449222088,
111      0.015490574762225151,
112      0.011365094222128391,
113      0.019505014643073082,
114      0.02123972401022911
115    ],
116    "n": 8
117  },
118  "comparison": {
119    "delta_mean": -0.047923396225087345,
120    "idea_wins": 8,
121    "n_pairs": 8,
122    "per_seed_diffs": [
123      -0.045856645330786705,
124      -0.04114901274442673,
125      -0.057636722922325134,
126      -0.044817524030804634,
127      -0.044461892917752266,
128      -0.07002124842256308,
129      -0.02654539979994297,
130      -0.052898723632097244
131    ],
132    "p_value": 0.0081,
133    "mde": 0.010652952386931449,
134    "mde_rel_pct": 15.96630390356262,
135    "verdict": "idea better (significant)",
136    "system_worked": true
137  },
138  "mechanism_signature": {
139    "prediction": "a gross token should shift Huber pooled quantiles less than arithmetic pooling",
140    "predicted_bounded_score": 0.06666666666666667,
141    "observed_huber_shift": 0.0727272778749466,
142    "observed_mean_shift": 0.8333333730697632,
143    "confirmed": true,
144    "custom_track": {
145      "name": "distribution_token_sequence",
146      "file": "hub_bench.py",
147      "domain": "sequence"
148    },
149    "idea_grid": [
150      {
151        "lr": 0.006,
152        "epochs": 12,
153        "delta": 0.5,
154        "mean": 0.020211950060911477
155      },
156      {
157        "lr": 0.006,
158        "epochs": 12,
159        "delta": 0.8,
160        "mean": 0.018798072007484734
161      },
162      {
163        "lr": 0.006,
164        "epochs": 12,
165        "delta": 1.2,
166        "mean": 0.020812171744182706
167      }
168    ],
169    "selected_idea_cfg": {
170      "lr": 0.006,
171      "epochs": 12,
172      "delta": 0.8
173    }
174  }
175}