1{
  2  "bench_version": 1,
  3  "track": "tabular",
  4  "model": "mlp_tiny",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "best_cfg": {
  9      "lr": 0.006,
 10      "wd": 0.0001
 11    },
 12    "sweep": [
 13      {
 14        "cfg": {
 15          "lr": 0.001,
 16          "wd": 0.0
 17        },
 18        "mean": 52.02932357788086
 19      },
 20      {
 21        "cfg": {
 22          "lr": 0.001,
 23          "wd": 0.0001
 24        },
 25        "mean": 51.98645782470703
 26      },
 27      {
 28        "cfg": {
 29          "lr": 0.003,
 30          "wd": 0.0
 31        },
 32        "mean": 14.625996589660645
 33      },
 34      {
 35        "cfg": {
 36          "lr": 0.003,
 37          "wd": 0.0001
 38        },
 39        "mean": 14.602366924285889
 40      },
 41      {
 42        "cfg": {
 43          "lr": 0.006,
 44          "wd": 0.0
 45        },
 46        "mean": 11.602271318435669
 47      },
 48      {
 49        "cfg": {
 50          "lr": 0.006,
 51          "wd": 0.0001
 52        },
 53        "mean": 11.320476055145264
 54      }
 55    ],
 56    "full": {
 57      "mean": 11.691799283027649,
 58      "std": 1.2449598187957995,
 59      "per_seed": [
 60        11.544059753417969,
 61        12.460237503051758,
 62        10.659576416015625,
 63        10.618030548095703,
 64        9.606558799743652,
 65        12.408422470092773,
 66        12.555534362792969,
 67        13.681974411010742
 68      ],
 69      "n": 8
 70    }
 71  },
 72  "idea": {
 73    "mean": 104.75726890563965,
 74    "std": 5.929301472950579,
 75    "per_seed": [
 76      102.7054443359375,
 77      102.88192749023438,
 78      106.44611358642578,
 79      95.48319244384766,
 80      116.45075988769531,
 81      102.46852111816406,
 82      101.3128662109375,
 83      110.309326171875
 84    ],
 85    "n": 8
 86  },
 87  "comparison": {
 88    "delta_mean": 93.065469622612,
 89    "idea_wins": 0,
 90    "n_pairs": 8,
 91    "per_seed_diffs": [
 92      91.16138458251953,
 93      90.42168998718262,
 94      95.78653717041016,
 95      84.86516189575195,
 96      106.84420108795166,
 97      90.06009864807129,
 98      88.75733184814453,
 99      96.62735176086426
100    ],
101    "p_value": 0.0081,
102    "mde": 5.612624544400254,
103    "mde_rel_pct": 48.00479728169638,
104    "verdict": "idea worse (significant)",
105    "system_worked": false
106  },
107  "idea_sweep": [
108    {
109      "cfg": {
110        "lr": 0.006,
111        "p": 2,
112        "gamma": 0.03
113      },
114      "mean": 101.87916946411133,
115      "n": 4
116    },
117    {
118      "cfg": {
119        "lr": 0.006,
120        "p": 2,
121        "gamma": 0.1
122      },
123      "mean": 157.33378219604492,
124      "n": 4
125    },
126    {
127      "cfg": {
128        "lr": 0.006,
129        "p": 2,
130        "gamma": 0.3
131      },
132      "mean": 173.4729461669922,
133      "n": 4
134    },
135    {
136      "cfg": {
137        "lr": 0.001,
138        "p": 4,
139        "gamma": 0.1
140      },
141      "mean": 233.65539169311523,
142      "n": 4
143    },
144    {
145      "cfg": {
146        "lr": 0.003,
147        "p": 4,
148        "gamma": 0.1
149      },
150      "mean": 227.60537338256836,
151      "n": 4
152    },
153    {
154      "cfg": {
155        "lr": 0.006,
156        "p": 4,
157        "gamma": 0.1
158      },
159      "mean": 206.5521469116211,
160      "n": 4
161    }
162  ],
163  "selection_cfg": {
164    "lr": 0.006,
165    "p": 2,
166    "gamma": 0.03
167  },
168  "equal_budget": {
169    "epochs": 16,
170    "batch": 128
171  },
172  "mechanism_signature": {
173    "quantity": "||G_HOME|| / ||raw DC gradient|| on minibatches",
174    "predicted": "smoothing should attenuate updates",
175    "observed_mean_ratio": 0.25457684118531804,
176    "observed_median_ratio": 0.22234734467118233,
177    "n": 80,
178    "confirmed": true
179  }
180}