Conditional OT barycenter feature augmentation / bench_report.json
Mechanism confirmed, baseline not beaten
1{
2 "bench_version": 1,
3 "track": "conditional_ot_domains",
4 "model": "shared_mlp_tiny",
5 "metric_direction": "lower is better",
6 "n_seeds": 8,
7 "baseline": {
8 "best_cfg": {
9 "lr": 0.006,
10 "eta": 0.15
11 },
12 "sweep": [
13 {
14 "cfg": {
15 "lr": 0.003,
16 "eta": 0.15
17 },
18 "mean": 0.07499999739229679
19 },
20 {
21 "cfg": {
22 "lr": 0.003,
23 "eta": 0.35
24 },
25 "mean": 0.07499999739229679
26 },
27 {
28 "cfg": {
29 "lr": 0.003,
30 "eta": 0.6
31 },
32 "mean": 0.07499999739229679
33 },
34 {
35 "cfg": {
36 "lr": 0.006,
37 "eta": 0.15
38 },
39 "mean": 0.07249999791383743
40 },
41 {
42 "cfg": {
43 "lr": 0.006,
44 "eta": 0.35
45 },
46 "mean": 0.07249999791383743
47 },
48 {
49 "cfg": {
50 "lr": 0.006,
51 "eta": 0.6
52 },
53 "mean": 0.07249999791383743
54 },
55 {
56 "cfg": {
57 "lr": 0.012,
58 "eta": 0.15
59 },
60 "mean": 0.08874999731779099
61 },
62 {
63 "cfg": {
64 "lr": 0.012,
65 "eta": 0.35
66 },
67 "mean": 0.08874999731779099
68 },
69 {
70 "cfg": {
71 "lr": 0.012,
72 "eta": 0.6
73 },
74 "mean": 0.08874999731779099
75 }
76 ],
77 "full": {
78 "mean": 0.07749999780207872,
79 "std": 0.010825316902066397,
80 "per_seed": [
81 0.07750000059604645,
82 0.07249999791383743,
83 0.07499999552965164,
84 0.06499999761581421,
85 0.08249999582767487,
86 0.0625,
87 0.08749999850988388,
88 0.09749999642372131
89 ],
90 "n": 8
91 }
92 },
93 "idea": {
94 "mean": 0.07249999837949872,
95 "std": 0.012437343083642742,
96 "per_seed": [
97 0.08500000089406967,
98 0.06749999523162842,
99 0.08249999582767487,
100 0.05249999836087227,
101 0.054999999701976776,
102 0.07750000059604645,
103 0.07249999791383743,
104 0.08749999850988388
105 ],
106 "n": 8
107 },
108 "comparison": {
109 "delta_mean": -0.004999999422580004,
110 "idea_wins": 5,
111 "n_pairs": 8,
112 "per_seed_diffs": [
113 0.007500000298023224,
114 -0.005000002682209015,
115 0.007500000298023224,
116 -0.01249999925494194,
117 -0.02749999612569809,
118 0.015000000596046448,
119 -0.015000000596046448,
120 -0.009999997913837433
121 ],
122 "p_value": 0.39075,
123 "mde": 0.011824999342373263,
124 "mde_rel_pct": 15.258064100301292,
125 "verdict": "no significant win",
126 "system_worked": false
127 },
128 "mechanism_signature": {
129 "mechanism_signature": {
130 "prediction": "three-source latent consensus has approximately one-third of pooled feature variance",
131 "predicted": 0.2850805396835009,
132 "observed": 0.26271023021803963,
133 "observed_reduction_ratio": 0.3066594176419761,
134 "confirmed": true,
135 "n_behavior_models": 8
136 },
137 "idea_sweep": [
138 {
139 "cfg": {
140 "lr": 0.003,
141 "eta": 0.15
142 },
143 "result": {
144 "mean": 0.07593749789521098,
145 "std": 0.016720940857393073,
146 "per_seed": [
147 0.07999999821186066,
148 0.07499999552965164,
149 0.10249999910593033,
150 0.044999998062849045,
151 0.057499997317790985,
152 0.08749999850988388,
153 0.07499999552965164,
154 0.08500000089406967
155 ],
156 "n": 8
157 }
158 },
159 {
160 "cfg": {
161 "lr": 0.003,
162 "eta": 0.35
163 },
164 "result": {
165 "mean": 0.07593749789521098,
166 "std": 0.016720940857393073,
167 "per_seed": [
168 0.07999999821186066,
169 0.07499999552965164,
170 0.10249999910593033,
171 0.044999998062849045,
172 0.057499997317790985,
173 0.08749999850988388,
174 0.07499999552965164,
175 0.08500000089406967
176 ],
177 "n": 8
178 }
179 },
180 {
181 "cfg": {
182 "lr": 0.003,
183 "eta": 0.6
184 },
185 "result": {
186 "mean": 0.07531249849125743,
187 "std": 0.01688367871052282,
188 "per_seed": [
189 0.07999999821186066,
190 0.07499999552965164,
191 0.10249999910593033,
192 0.044999998062849045,
193 0.054999999701976776,
194 0.08500000089406967,
195 0.07499999552965164,
196 0.08500000089406967
197 ],
198 "n": 8
199 }
200 },
201 {
202 "cfg": {
203 "lr": 0.006,
204 "eta": 0.15
205 },
206 "result": {
207 "mean": 0.07499999832361937,
208 "std": 0.012990381393724975,
209 "per_seed": [
210 0.08749999850988388,
211 0.07499999552965164,
212 0.07499999552965164,
213 0.04749999940395355,
214 0.07999999821186066,
215 0.06499999761581421,
216 0.07750000059604645,
217 0.0925000011920929
218 ],
219 "n": 8
220 }
221 },
222 {
223 "cfg": {
224 "lr": 0.006,
225 "eta": 0.35
226 },
227 "result": {
228 "mean": 0.07749999780207872,
229 "std": 0.014361406700648869,
230 "per_seed": [
231 0.08249999582767487,
232 0.07499999552965164,
233 0.08249999582767487,
234 0.04749999940395355,
235 0.0925000011920929,
236 0.06499999761581421,
237 0.07999999821186066,
238 0.0949999988079071
239 ],
240 "n": 8
241 }
242 },
243 {
244 "cfg": {
245 "lr": 0.006,
246 "eta": 0.6
247 },
248 "result": {
249 "mean": 0.07749999780207872,
250 "std": 0.01499999950950353,
251 "per_seed": [
252 0.08249999582767487,
253 0.07499999552965164,
254 0.07750000059604645,
255 0.04749999940395355,
256 0.09749999642372131,
257 0.06499999761581421,
258 0.07999999821186066,
259 0.0949999988079071
260 ],
261 "n": 8
262 }
263 },
264 {
265 "cfg": {
266 "lr": 0.012,
267 "eta": 0.15
268 },
269 "result": {
270 "mean": 0.07249999837949872,
271 "std": 0.012437343083642742,
272 "per_seed": [
273 0.08500000089406967,
274 0.06749999523162842,
275 0.08249999582767487,
276 0.05249999836087227,
277 0.054999999701976776,
278 0.07750000059604645,
279 0.07249999791383743,
280 0.08749999850988388
281 ],
282 "n": 8
283 }
284 },
285 {
286 "cfg": {
287 "lr": 0.012,
288 "eta": 0.35
289 },
290 "result": {
291 "mean": 0.07562499865889549,
292 "std": 0.013506364375069949,
293 "per_seed": [
294 0.0949999988079071,
295 0.07249999791383743,
296 0.07249999791383743,
297 0.05249999836087227,
298 0.05999999865889549,
299 0.08749999850988388,
300 0.07750000059604645,
301 0.08749999850988388
302 ],
303 "n": 8
304 }
305 },
306 {
307 "cfg": {
308 "lr": 0.012,
309 "eta": 0.6
310 },
311 "result": {
312 "mean": 0.0796874975785613,
313 "std": 0.01454720197649073,
314 "per_seed": [
315 0.0949999988079071,
316 0.07249999791383743,
317 0.07000000029802322,
318 0.04999999701976776,
319 0.07999999821186066,
320 0.09749999642372131,
321 0.08249999582767487,
322 0.08999999612569809
323 ],
324 "n": 8
325 }
326 }
327 ],
328 "custom_track": {
329 "name": "conditional_ot_domains",
330 "file": "conditional_ot_track.py",
331 "domain": "domain_generalization"
332 },
333 "protocol": {
334 "paired_seeds": [
335 0,
336 1,
337 2,
338 3,
339 4,
340 5,
341 6,
342 7
343 ],
344 "sweep_seeds": [
345 0,
346 1,
347 2,
348 3
349 ],
350 "grid_union": [
351 {
352 "lr": 0.003,
353 "eta": 0.15
354 },
355 {
356 "lr": 0.003,
357 "eta": 0.35
358 },
359 {
360 "lr": 0.003,
361 "eta": 0.6
362 },
363 {
364 "lr": 0.006,
365 "eta": 0.15
366 },
367 {
368 "lr": 0.006,
369 "eta": 0.35
370 },
371 {
372 "lr": 0.006,
373 "eta": 0.6
374 },
375 {
376 "lr": 0.012,
377 "eta": 0.15
378 },
379 {
380 "lr": 0.012,
381 "eta": 0.35
382 },
383 {
384 "lr": 0.012,
385 "eta": 0.6
386 }
387 ],
388 "baseline_tuned_and_reevaluated": true,
389 "task_match": "multi-source domain generalization with latent feature consensus"
390 }
391 }
392}