1{
  2  "bench_version": 1,
  3  "track": "unordered_pointset_denoising",
  4  "model": "mlp_tiny",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "best_cfg": {
  9      "lr": 0.003,
 10      "weight_decay": 0.0001
 11    },
 12    "sweep": [
 13      {
 14        "cfg": {
 15          "lr": 0.001,
 16          "weight_decay": 0.0
 17        },
 18        "mean": 0.09564068168401718
 19      },
 20      {
 21        "cfg": {
 22          "lr": 0.001,
 23          "weight_decay": 0.0001
 24        },
 25        "mean": 0.09560250677168369
 26      },
 27      {
 28        "cfg": {
 29          "lr": 0.002,
 30          "weight_decay": 0.0
 31        },
 32        "mean": 0.084096509963274
 33      },
 34      {
 35        "cfg": {
 36          "lr": 0.002,
 37          "weight_decay": 0.0001
 38        },
 39        "mean": 0.0841062506660819
 40      },
 41      {
 42        "cfg": {
 43          "lr": 0.003,
 44          "weight_decay": 0.0
 45        },
 46        "mean": 0.07198812533169985
 47      },
 48      {
 49        "cfg": {
 50          "lr": 0.003,
 51          "weight_decay": 0.0001
 52        },
 53        "mean": 0.07177905738353729
 54      }
 55    ],
 56    "full": {
 57      "mean": 0.07177905738353729,
 58      "std": 0.0038503444468855823,
 59      "per_seed": [
 60        0.07469823211431503,
 61        0.06881649047136307,
 62        0.06867200881242752,
 63        0.0654606819152832,
 64        0.0715550109744072,
 65        0.07852364331483841,
 66        0.07229366153478622,
 67        0.07421272993087769
 68      ],
 69      "n": 8,
 70      "collision_per_seed": [
 71        0.054999999701976776,
 72        0.042500000447034836,
 73        0.05374999716877937,
 74        0.06499999761581421,
 75        0.0637499988079071,
 76        0.0625,
 77        0.04625000059604645,
 78        0.07750000059604645
 79      ],
 80      "collision_mean": 0.05828124936670065,
 81      "mean_gap": 0.9999973326921463,
 82      "cfg": {
 83        "lr": 0.003,
 84        "weight_decay": 0.0001
 85      }
 86    }
 87  },
 88  "idea": {
 89    "mean": 0.08495018165558577,
 90    "std": 0.0036923579732418346,
 91    "per_seed": [
 92      0.08852718770503998,
 93      0.08314542472362518,
 94      0.08323974907398224,
 95      0.0792478397488594,
 96      0.0855635404586792,
 97      0.09225999563932419,
 98      0.08457005023956299,
 99      0.08304766565561295
100    ],
101    "n": 8,
102    "collision_per_seed": [
103      0.07499999552965164,
104      0.05624999850988388,
105      0.05624999850988388,
106      0.07374999672174454,
107      0.05999999865889549,
108      0.07000000029802322,
109      0.0625,
110      0.06875000149011612
111    ],
112    "collision_mean": 0.06531249871477485,
113    "mean_gap": 0.9999975040555,
114    "cfg": {
115      "lr": 0.003,
116      "weight_decay": 0.0001,
117      "lam": 0.1,
118      "beta": 2.0,
119      "delta": 0.08
120    }
121  },
122  "comparison": {
123    "delta_mean": 0.013171124272048473,
124    "idea_wins": 0,
125    "n_pairs": 8,
126    "per_seed_diffs": [
127      0.013828955590724945,
128      0.014328934252262115,
129      0.014567740261554718,
130      0.013787157833576202,
131      0.014008529484272003,
132      0.013736352324485779,
133      0.012276388704776764,
134      0.00883493572473526
135    ],
136    "p_value": 0.0081,
137    "verdict": "idea worse (significant)",
138    "system_worked": false
139  },
140  "custom_track": {
141    "name": "unordered_pointset_denoising",
142    "file": "pointset_fusion_track.py",
143    "domain": "point-set-diffusion"
144  },
145  "idea_settings": [
146    {
147      "mean": 0.08495018165558577,
148      "std": 0.0036923579732418346,
149      "per_seed": [
150        0.08852718770503998,
151        0.08314542472362518,
152        0.08323974907398224,
153        0.0792478397488594,
154        0.0855635404586792,
155        0.09225999563932419,
156        0.08457005023956299,
157        0.08304766565561295
158      ],
159      "n": 8,
160      "collision_per_seed": [
161        0.07499999552965164,
162        0.05624999850988388,
163        0.05624999850988388,
164        0.07374999672174454,
165        0.05999999865889549,
166        0.07000000029802322,
167        0.0625,
168        0.06875000149011612
169      ],
170      "collision_mean": 0.06531249871477485,
171      "mean_gap": 0.9999975040555,
172      "cfg": {
173        "lr": 0.003,
174        "weight_decay": 0.0001,
175        "lam": 0.1,
176        "beta": 2.0,
177        "delta": 0.08
178      }
179    },
180    {
181      "mean": 0.0936331320554018,
182      "std": 0.0037630771536154607,
183      "per_seed": [
184        0.09448400139808655,
185        0.09308362752199173,
186        0.08933884650468826,
187        0.0864451453089714,
188        0.09648385643959045,
189        0.09782309830188751,
190        0.09371592104434967,
191        0.09769055992364883
192      ],
193      "n": 8,
194      "collision_per_seed": [
195        0.07999999821186066,
196        0.07000000029802322,
197        0.06749999523162842,
198        0.07124999910593033,
199        0.0612499974668026,
200        0.07249999791383743,
201        0.0637499988079071,
202        0.06499999761581421
203      ],
204      "collision_mean": 0.0689062480814755,
205      "mean_gap": 0.9999975264072418,
206      "cfg": {
207        "lr": 0.002,
208        "weight_decay": 0.0001,
209        "lam": 0.1,
210        "beta": 2.0,
211        "delta": 0.08
212      }
213    },
214    {
215      "mean": 0.09946490358561277,
216      "std": 0.0037473580173809696,
217      "per_seed": [
218        0.09996496886014938,
219        0.10018042474985123,
220        0.09583928436040878,
221        0.09132290631532669,
222        0.1030997484922409,
223        0.10119754076004028,
224        0.1009417474269867,
225        0.10317260771989822
226      ],
227      "n": 8,
228      "collision_per_seed": [
229        0.07750000059604645,
230        0.08374999463558197,
231        0.05874999985098839,
232        0.06499999761581421,
233        0.06749999523162842,
234        0.07249999791383743,
235        0.08249999582767487,
236        0.07625000178813934
237      ],
238      "collision_mean": 0.07296874793246388,
239      "mean_gap": 0.9999975264072418,
240      "cfg": {
241        "lr": 0.001,
242        "weight_decay": 0.0001,
243        "lam": 0.1,
244        "beta": 2.0,
245        "delta": 0.08
246      }
247    }
248  ],
249  "mechanism_signature": {
250    "quantity": "normalized predicted short-gap collision rate",
251    "baseline_predicted_collision": 0.05828124936670065,
252    "idea_predicted_collision": 0.06531249871477485,
253    "predicted_delta": 0.007031249348074198,
254    "observed_test_mse_delta": 0.013171124272048473,
255    "confirmed": false
256  }
257}