1{
  2  "bench_version": 1,
  3  "track": "tabular",
  4  "model": "mlp_tiny",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "best_cfg": {
  9      "lr": 0.01
 10    },
 11    "sweep": [
 12      {
 13        "cfg": {
 14          "lr": 0.001
 15        },
 16        "mean": 18.323792934417725
 17      },
 18      {
 19        "cfg": {
 20          "lr": 0.003
 21        },
 22        "mean": 12.731719493865967
 23      },
 24      {
 25        "cfg": {
 26          "lr": 0.01
 27        },
 28        "mean": 7.733822226524353
 29      }
 30    ],
 31    "full": {
 32      "mean": 7.905456006526947,
 33      "std": 0.7333165741197974,
 34      "per_seed": [
 35        7.984025478363037,
 36        6.387940406799316,
 37        7.856008529663086,
 38        8.707314491271973,
 39        8.079574584960938,
 40        7.502847671508789,
 41        8.969817161560059,
 42        7.756119728088379
 43      ],
 44      "n": 8
 45    },
 46    "all_union_rates": {
 47      "0.001": {
 48        "mean": 19.335745573043823,
 49        "std": 3.657827404086414,
 50        "per_seed": [
 51          25.777250289916992,
 52          13.66278076171875,
 53          17.315614700317383,
 54          16.539525985717773,
 55          18.554529190063477,
 56          18.02096939086914,
 57          22.761363983154297,
 58          22.053930282592773
 59        ],
 60        "n": 8
 61      },
 62      "0.003": {
 63        "mean": 13.304376482963562,
 64        "std": 1.3247165411283601,
 65        "per_seed": [
 66          14.347658157348633,
 67          10.474533081054688,
 68          13.201133728027344,
 69          12.903553009033203,
 70          13.815519332885742,
 71          13.35650634765625,
 72          12.940177917480469,
 73          15.395930290222168
 74        ],
 75        "n": 8
 76      },
 77      "0.01": {
 78        "mean": 7.905456006526947,
 79        "std": 0.7333165741197974,
 80        "per_seed": [
 81          7.984025478363037,
 82          6.387940406799316,
 83          7.856008529663086,
 84          8.707314491271973,
 85          8.079574584960938,
 86          7.502847671508789,
 87          8.969817161560059,
 88          7.756119728088379
 89        ],
 90        "n": 8
 91      }
 92    }
 93  },
 94  "idea": {
 95    "mean": 11.219475030899048,
 96    "std": 0.9330729672086647,
 97    "per_seed": [
 98      11.737688064575195,
 99      9.196637153625488,
100      10.628690719604492,
101      11.08810806274414,
102      11.466021537780762,
103      11.766623497009277,
104      11.28360366821289,
105      12.588427543640137
106    ],
107    "n": 8
108  },
109  "comparison": {
110    "delta_mean": 3.314019024372101,
111    "idea_wins": 0,
112    "n_pairs": 8,
113    "per_seed_diffs": [
114      3.753662586212158,
115      2.808696746826172,
116      2.7726821899414062,
117      2.380793571472168,
118      3.386446952819824,
119      4.263775825500488,
120      2.313786506652832,
121      4.832307815551758
122    ],
123    "p_value": 0.0081,
124    "mde": 0.7627905507582391,
125    "mde_rel_pct": 9.648912727215984,
126    "verdict": "idea worse (significant)",
127    "system_worked": false
128  },
129  "mechanism_signature": {
130    "prediction": "weakly supported coordinates receive smaller update metric",
131    "observed": {
132      "support_fraction": 0.514455258846283,
133      "q_mean": 0.4733452796936035,
134      "grad_norm": 44.26974844932556
135    },
136    "confirmed": true
137  },
138  "idea_all_union_rates": {
139    "0.001": {
140      "mean": 198.77429580688477,
141      "std": 13.133327131145185,
142      "per_seed": [
143        210.48277282714844,
144        190.67105102539062,
145        208.48660278320312,
146        183.82296752929688,
147        205.18133544921875,
148        176.4065399169922,
149        198.54098510742188,
150        216.60211181640625
151      ],
152      "n": 8
153    },
154    "0.003": {
155      "mean": 39.026103258132935,
156      "std": 9.54746173172683,
157      "per_seed": [
158        48.3967170715332,
159        26.32292938232422,
160        37.33447265625,
161        32.64558410644531,
162        41.14009094238281,
163        27.771005630493164,
164        56.16447067260742,
165        42.433555603027344
166      ],
167      "n": 8
168    },
169    "0.01": {
170      "mean": 11.219475030899048,
171      "std": 0.9330729672086647,
172      "per_seed": [
173        11.737688064575195,
174        9.196637153625488,
175        10.628690719604492,
176        11.08810806274414,
177        11.466021537780762,
178        11.766623497009277,
179        11.28360366821289,
180        12.588427543640137
181      ],
182      "n": 8
183    }
184  },
185  "idea_best_lr": 0.01
186}