1{
  2  "track": "local_tabular_low_rank_regression",
  3  "protocol_status": "fixed bench unavailable at /home/maxwelhelp/all/math2nn/bench; this is supplemental only",
  4  "baseline_sweep": [
  5    {
  6      "lr": 0.001,
  7      "baseline_mean": 0.3229888416826725,
  8      "idea_mean": 0.32168807089328766,
  9      "delta_mean": -0.001300770789384842,
 10      "p_value": 0.2459
 11    },
 12    {
 13      "lr": 0.003,
 14      "baseline_mean": 0.30325367487967014,
 15      "idea_mean": 0.3026796840131283,
 16      "delta_mean": -0.0005739908665418625,
 17      "p_value": 0.2211
 18    },
 19    {
 20      "lr": 0.006,
 21      "baseline_mean": 0.3067930955439806,
 22      "idea_mean": 0.3066288847476244,
 23      "delta_mean": -0.00016421079635620117,
 24      "p_value": 0.375
 25    }
 26  ],
 27  "best_baseline": {
 28    "lr": 0.003,
 29    "baseline_mean": 0.30325367487967014,
 30    "idea_mean": 0.3026796840131283,
 31    "delta_mean": -0.0005739908665418625,
 32    "p_value": 0.2211
 33  },
 34  "best_idea": {
 35    "lr": 0.003,
 36    "baseline_mean": 0.30325367487967014,
 37    "idea_mean": 0.3026796840131283,
 38    "delta_mean": -0.0005739908665418625,
 39    "p_value": 0.2211
 40  },
 41  "paired_results": [
 42    {
 43      "seed": 0,
 44      "lr": 0.001,
 45      "baseline": {
 46        "mse": 0.29650869965553284,
 47        "predicted_retained_proxy": 0.1937541961669922,
 48        "observed_retained_response": 0.5846818760037422,
 49        "rank": 6
 50      },
 51      "idea": {
 52        "mse": 0.29356062412261963,
 53        "predicted_retained_proxy": 0.15797016024589539,
 54        "observed_retained_response": 0.5918992236256599,
 55        "rank": 6
 56      },
 57      "delta": -0.002948075532913208
 58    },
 59    {
 60      "seed": 1,
 61      "lr": 0.001,
 62      "baseline": {
 63        "mse": 0.2703535556793213,
 64        "predicted_retained_proxy": 0.23858273029327393,
 65        "observed_retained_response": 0.5590907707810402,
 66        "rank": 6
 67      },
 68      "idea": {
 69        "mse": 0.2698759138584137,
 70        "predicted_retained_proxy": 0.23667477071285248,
 71        "observed_retained_response": 0.5697524696588516,
 72        "rank": 6
 73      },
 74      "delta": -0.0004776418209075928
 75    },
 76    {
 77      "seed": 2,
 78      "lr": 0.001,
 79      "baseline": {
 80        "mse": 0.3012450337409973,
 81        "predicted_retained_proxy": 0.16559083759784698,
 82        "observed_retained_response": 0.564979299902916,
 83        "rank": 6
 84      },
 85      "idea": {
 86        "mse": 0.290658175945282,
 87        "predicted_retained_proxy": 0.11861691623926163,
 88        "observed_retained_response": 0.5777160674333572,
 89        "rank": 6
 90      },
 91      "delta": -0.010586857795715332
 92    },
 93    {
 94      "seed": 3,
 95      "lr": 0.001,
 96      "baseline": {
 97        "mse": 0.30093708634376526,
 98        "predicted_retained_proxy": 0.13248607516288757,
 99        "observed_retained_response": 0.6169104352593422,
100        "rank": 6
101      },
102      "idea": {
103        "mse": 0.3020533621311188,
104        "predicted_retained_proxy": 0.09665870666503906,
105        "observed_retained_response": 0.6377759277820587,
106        "rank": 6
107      },
108      "delta": 0.0011162757873535156
109    },
110    {
111      "seed": 4,
112      "lr": 0.001,
113      "baseline": {
114        "mse": 0.2939927875995636,
115        "predicted_retained_proxy": 0.1950964331626892,
116        "observed_retained_response": 0.5940887182950974,
117        "rank": 6
118      },
119      "idea": {
120        "mse": 0.29432255029678345,
121        "predicted_retained_proxy": 0.18268471956253052,
122        "observed_retained_response": 0.6008488908410072,
123        "rank": 6
124      },
125      "delta": 0.00032976269721984863
126    },
127    {
128      "seed": 5,
129      "lr": 0.001,
130      "baseline": {
131        "mse": 0.5648365020751953,
132        "predicted_retained_proxy": 0.21797959506511688,
133        "observed_retained_response": 0.47909506037831306,
134        "rank": 6
135      },
136      "idea": {
137        "mse": 0.5733405351638794,
138        "predicted_retained_proxy": 0.1479392945766449,
139        "observed_retained_response": 0.4866265542805195,
140        "rank": 6
141      },
142      "delta": 0.008504033088684082
143    },
144    {
145      "seed": 6,
146      "lr": 0.001,
147      "baseline": {
148        "mse": 0.32254257798194885,
149        "predicted_retained_proxy": 0.23743118345737457,
150        "observed_retained_response": 0.5972476936876774,
151        "rank": 6
152      },
153      "idea": {
154        "mse": 0.318254679441452,
155        "predicted_retained_proxy": 0.18390978872776031,
156        "observed_retained_response": 0.6131080314517021,
157        "rank": 6
158      },
159      "delta": -0.004287898540496826
160    },
161    {
162      "seed": 7,
163      "lr": 0.001,
164      "baseline": {
165        "mse": 0.23349449038505554,
166        "predicted_retained_proxy": 0.1556420624256134,
167        "observed_retained_response": 0.5693437606096268,
168        "rank": 6
169      },
170      "idea": {
171        "mse": 0.23143872618675232,
172        "predicted_retained_proxy": 0.13866308331489563,
173        "observed_retained_response": 0.5724697560071945,
174        "rank": 6
175      },
176      "delta": -0.0020557641983032227
177    },
178    {
179      "seed": 0,
180      "lr": 0.003,
181      "baseline": {
182        "mse": 0.28676044940948486,
183        "predicted_retained_proxy": 0.08567658811807632,
184        "observed_retained_response": 0.6662804409861565,
185        "rank": 6
186      },
187      "idea": {
188        "mse": 0.28411293029785156,
189        "predicted_retained_proxy": 0.055248551070690155,
190        "observed_retained_response": 0.6709468439221382,
191        "rank": 6
192      },
193      "delta": -0.0026475191116333008
194    },
195    {
196      "seed": 1,
197      "lr": 0.003,
198      "baseline": {
199        "mse": 0.24638411402702332,
200        "predicted_retained_proxy": 0.15895552933216095,
201        "observed_retained_response": 0.6180978156626225,
202        "rank": 6
203      },
204      "idea": {
205        "mse": 0.24585376679897308,
206        "predicted_retained_proxy": 0.07616078108549118,
207        "observed_retained_response": 0.5821574740111828,
208        "rank": 6
209      },
210      "delta": -0.0005303472280502319
211    },
212    {
213      "seed": 2,
214      "lr": 0.003,
215      "baseline": {
216        "mse": 0.3136810064315796,
217        "predicted_retained_proxy": 0.3495432734489441,
218        "observed_retained_response": 0.6892230659723282,
219        "rank": 6
220      },
221      "idea": {
222        "mse": 0.3149034380912781,
223        "predicted_retained_proxy": 0.196433886885643,
224        "observed_retained_response": 0.6687191724777222,
225        "rank": 6
226      },
227      "delta": 0.0012224316596984863
228    },
229    {
230      "seed": 3,
231      "lr": 0.003,
232      "baseline": {
233        "mse": 0.2992812395095825,
234        "predicted_retained_proxy": 0.11782639473676682,
235        "observed_retained_response": 0.6399654373526573,
236        "rank": 6
237      },
238      "idea": {
239        "mse": 0.2992812395095825,
240        "predicted_retained_proxy": 0.11782639473676682,
241        "observed_retained_response": 0.6399654373526573,
242        "rank": 6
243      },
244      "delta": 0.0
245    },
246    {
247      "seed": 4,
248      "lr": 0.003,
249      "baseline": {
250        "mse": 0.27185118198394775,
251        "predicted_retained_proxy": 0.11754812300205231,
252        "observed_retained_response": 0.6420586034655571,
253        "rank": 6
254      },
255      "idea": {
256        "mse": 0.27225157618522644,
257        "predicted_retained_proxy": 0.10567549616098404,
258        "observed_retained_response": 0.6364488303661346,
259        "rank": 6
260      },
261      "delta": 0.0004003942012786865
262    },
263    {
264      "seed": 5,
265      "lr": 0.003,
266      "baseline": {
267        "mse": 0.5052655935287476,
268        "predicted_retained_proxy": 0.17139050364494324,
269        "observed_retained_response": 0.6117693334817886,
270        "rank": 6
271      },
272      "idea": {
273        "mse": 0.5055328011512756,
274        "predicted_retained_proxy": 0.15541502833366394,
275        "observed_retained_response": 0.6354210749268532,
276        "rank": 6
277      },
278      "delta": 0.00026720762252807617
279    },
280    {
281      "seed": 6,
282      "lr": 0.003,
283      "baseline": {
284        "mse": 0.2950036823749542,
285        "predicted_retained_proxy": 0.15834441781044006,
286        "observed_retained_response": 0.6878739595413208,
287        "rank": 6
288      },
289      "idea": {
290        "mse": 0.2912462651729584,
291        "predicted_retained_proxy": 0.13159003853797913,
292        "observed_retained_response": 0.6799127534031868,
293        "rank": 6
294      },
295      "delta": -0.0037574172019958496
296    },
297    {
298      "seed": 7,
299      "lr": 0.003,
300      "baseline": {
301        "mse": 0.20780213177204132,
302        "predicted_retained_proxy": 0.07717398554086685,
303        "observed_retained_response": 0.611984059214592,
304        "rank": 6
305      },
306      "idea": {
307        "mse": 0.20825545489788055,
308        "predicted_retained_proxy": 0.05258169770240784,
309        "observed_retained_response": 0.5936216339468956,
310        "rank": 6
311      },
312      "delta": 0.0004533231258392334
313    },
314    {
315      "seed": 0,
316      "lr": 0.006,
317      "baseline": {
318        "mse": 0.2900365889072418,
319        "predicted_retained_proxy": 0.08479925990104675,
320        "observed_retained_response": 0.6431076973676682,
321        "rank": 6
322      },
323      "idea": {
324        "mse": 0.2889346182346344,
325        "predicted_retained_proxy": 0.07691498100757599,
326        "observed_retained_response": 0.6423569396138191,
327        "rank": 6
328      },
329      "delta": -0.0011019706726074219
330    },
331    {
332      "seed": 1,
333      "lr": 0.006,
334      "baseline": {
335        "mse": 0.2498152107000351,
336        "predicted_retained_proxy": 0.16318391263484955,
337        "observed_retained_response": 0.6058170236647129,
338        "rank": 6
339      },
340      "idea": {
341        "mse": 0.2503966987133026,
342        "predicted_retained_proxy": 0.15665803849697113,
343        "observed_retained_response": 0.6152011044323444,
344        "rank": 6
345      },
346      "delta": 0.0005814880132675171
347    },
348    {
349      "seed": 2,
350      "lr": 0.006,
351      "baseline": {
352        "mse": 0.3184656798839569,
353        "predicted_retained_proxy": 0.34392213821411133,
354        "observed_retained_response": 0.6691530644893646,
355        "rank": 6
356      },
357      "idea": {
358        "mse": 0.3190155625343323,
359        "predicted_retained_proxy": 0.18742811679840088,
360        "observed_retained_response": 0.6492236107587814,
361        "rank": 6
362      },
363      "delta": 0.0005498826503753662
364    },
365    {
366      "seed": 3,
367      "lr": 0.006,
368      "baseline": {
369        "mse": 0.3039996325969696,
370        "predicted_retained_proxy": 0.10392030328512192,
371        "observed_retained_response": 0.6193787045776844,
372        "rank": 6
373      },
374      "idea": {
375        "mse": 0.3013850748538971,
376        "predicted_retained_proxy": 0.10185642540454865,
377        "observed_retained_response": 0.6289432123303413,
378        "rank": 6
379      },
380      "delta": -0.0026145577430725098
381    },
382    {
383      "seed": 4,
384      "lr": 0.006,
385      "baseline": {
386        "mse": 0.27291032671928406,
387        "predicted_retained_proxy": 0.11452590674161911,
388        "observed_retained_response": 0.6283508837223053,
389        "rank": 6
390      },
391      "idea": {
392        "mse": 0.27344390749931335,
393        "predicted_retained_proxy": 0.10965780168771744,
394        "observed_retained_response": 0.621874064207077,
395        "rank": 6
396      },
397      "delta": 0.0005335807800292969
398    },
399    {
400      "seed": 5,
401      "lr": 0.006,
402      "baseline": {
403        "mse": 0.5082470774650574,
404        "predicted_retained_proxy": 0.1677103191614151,
405        "observed_retained_response": 0.577905185520649,
406        "rank": 6
407      },
408      "idea": {
409        "mse": 0.5096269249916077,
410        "predicted_retained_proxy": 0.1527785360813141,
411        "observed_retained_response": 0.6050795763731003,
412        "rank": 6
413      },
414      "delta": 0.001379847526550293
415    },
416    {
417      "seed": 6,
418      "lr": 0.006,
419      "baseline": {
420        "mse": 0.29952022433280945,
421        "predicted_retained_proxy": 0.15662352740764618,
422        "observed_retained_response": 0.6685446798801422,
423        "rank": 6
424      },
425      "idea": {
426        "mse": 0.29916277527809143,
427        "predicted_retained_proxy": 0.1364256590604782,
428        "observed_retained_response": 0.6665552407503128,
429        "rank": 6
430      },
431      "delta": -0.0003574490547180176
432    },
433    {
434      "seed": 7,
435      "lr": 0.006,
436      "baseline": {
437        "mse": 0.21135002374649048,
438        "predicted_retained_proxy": 0.0768255814909935,
439        "observed_retained_response": 0.5843467249069363,
440        "rank": 6
441      },
442      "idea": {
443        "mse": 0.21106551587581635,
444        "predicted_retained_proxy": 0.06295523047447205,
445        "observed_retained_response": 0.59474778175354,
446        "rank": 6
447      },
448      "delta": -0.0002845078706741333
449    }
450  ],
451  "bench_report": {
452    "baseline_sweep": [
453      {
454        "lr": 0.001,
455        "baseline_mean": 0.3229888416826725,
456        "idea_mean": 0.32168807089328766,
457        "delta_mean": -0.001300770789384842,
458        "p_value": 0.2459
459      },
460      {
461        "lr": 0.003,
462        "baseline_mean": 0.30325367487967014,
463        "idea_mean": 0.3026796840131283,
464        "delta_mean": -0.0005739908665418625,
465        "p_value": 0.2211
466      },
467      {
468        "lr": 0.006,
469        "baseline_mean": 0.3067930955439806,
470        "idea_mean": 0.3066288847476244,
471        "delta_mean": -0.00016421079635620117,
472        "p_value": 0.375
473      }
474    ],
475    "idea_per_seed_results": [
476      {
477        "seed": 0,
478        "lr": 0.003,
479        "baseline": {
480          "mse": 0.28676044940948486,
481          "predicted_retained_proxy": 0.08567658811807632,
482          "observed_retained_response": 0.6662804409861565,
483          "rank": 6
484        },
485        "idea": {
486          "mse": 0.28411293029785156,
487          "predicted_retained_proxy": 0.055248551070690155,
488          "observed_retained_response": 0.6709468439221382,
489          "rank": 6
490        },
491        "delta": -0.0026475191116333008
492      },
493      {
494        "seed": 1,
495        "lr": 0.003,
496        "baseline": {
497          "mse": 0.24638411402702332,
498          "predicted_retained_proxy": 0.15895552933216095,
499          "observed_retained_response": 0.6180978156626225,
500          "rank": 6
501        },
502        "idea": {
503          "mse": 0.24585376679897308,
504          "predicted_retained_proxy": 0.07616078108549118,
505          "observed_retained_response": 0.5821574740111828,
506          "rank": 6
507        },
508        "delta": -0.0005303472280502319
509      },
510      {
511        "seed": 2,
512        "lr": 0.003,
513        "baseline": {
514          "mse": 0.3136810064315796,
515          "predicted_retained_proxy": 0.3495432734489441,
516          "observed_retained_response": 0.6892230659723282,
517          "rank": 6
518        },
519        "idea": {
520          "mse": 0.3149034380912781,
521          "predicted_retained_proxy": 0.196433886885643,
522          "observed_retained_response": 0.6687191724777222,
523          "rank": 6
524        },
525        "delta": 0.0012224316596984863
526      },
527      {
528        "seed": 3,
529        "lr": 0.003,
530        "baseline": {
531          "mse": 0.2992812395095825,
532          "predicted_retained_proxy": 0.11782639473676682,
533          "observed_retained_response": 0.6399654373526573,
534          "rank": 6
535        },
536        "idea": {
537          "mse": 0.2992812395095825,
538          "predicted_retained_proxy": 0.11782639473676682,
539          "observed_retained_response": 0.6399654373526573,
540          "rank": 6
541        },
542        "delta": 0.0
543      },
544      {
545        "seed": 4,
546        "lr": 0.003,
547        "baseline": {
548          "mse": 0.27185118198394775,
549          "predicted_retained_proxy": 0.11754812300205231,
550          "observed_retained_response": 0.6420586034655571,
551          "rank": 6
552        },
553        "idea": {
554          "mse": 0.27225157618522644,
555          "predicted_retained_proxy": 0.10567549616098404,
556          "observed_retained_response": 0.6364488303661346,
557          "rank": 6
558        },
559        "delta": 0.0004003942012786865
560      },
561      {
562        "seed": 5,
563        "lr": 0.003,
564        "baseline": {
565          "mse": 0.5052655935287476,
566          "predicted_retained_proxy": 0.17139050364494324,
567          "observed_retained_response": 0.6117693334817886,
568          "rank": 6
569        },
570        "idea": {
571          "mse": 0.5055328011512756,
572          "predicted_retained_proxy": 0.15541502833366394,
573          "observed_retained_response": 0.6354210749268532,
574          "rank": 6
575        },
576        "delta": 0.00026720762252807617
577      },
578      {
579        "seed": 6,
580        "lr": 0.003,
581        "baseline": {
582          "mse": 0.2950036823749542,
583          "predicted_retained_proxy": 0.15834441781044006,
584          "observed_retained_response": 0.6878739595413208,
585          "rank": 6
586        },
587        "idea": {
588          "mse": 0.2912462651729584,
589          "predicted_retained_proxy": 0.13159003853797913,
590          "observed_retained_response": 0.6799127534031868,
591          "rank": 6
592        },
593        "delta": -0.0037574172019958496
594      },
595      {
596        "seed": 7,
597        "lr": 0.003,
598        "baseline": {
599          "mse": 0.20780213177204132,
600          "predicted_retained_proxy": 0.07717398554086685,
601          "observed_retained_response": 0.611984059214592,
602          "rank": 6
603        },
604        "idea": {
605          "mse": 0.20825545489788055,
606          "predicted_retained_proxy": 0.05258169770240784,
607          "observed_retained_response": 0.5936216339468956,
608          "rank": 6
609        },
610        "delta": 0.0004533231258392334
611      }
612    ],
613    "paired_delta_mean": -0.0005739908665418625,
614    "permutation_p_value": 0.2211
615  },
616  "mechanism_signature": {
617    "predicted": "retained rank-one first-order contribution proxy",
618    "observed": "mean absolute retained rank-one response on held-out inputs",
619    "predicted_values": [
620      0.15797016024589539,
621      0.23667477071285248,
622      0.11861691623926163,
623      0.09665870666503906,
624      0.18268471956253052,
625      0.1479392945766449,
626      0.18390978872776031,
627      0.13866308331489563,
628      0.055248551070690155,
629      0.07616078108549118,
630      0.196433886885643,
631      0.11782639473676682,
632      0.10567549616098404,
633      0.15541502833366394,
634      0.13159003853797913,
635      0.05258169770240784,
636      0.07691498100757599,
637      0.15665803849697113,
638      0.18742811679840088,
639      0.10185642540454865,
640      0.10965780168771744,
641      0.1527785360813141,
642      0.1364256590604782,
643      0.06295523047447205
644    ],
645    "observed_values": [
646      0.5918992236256599,
647      0.5697524696588516,
648      0.5777160674333572,
649      0.6377759277820587,
650      0.6008488908410072,
651      0.4866265542805195,
652      0.6131080314517021,
653      0.5724697560071945,
654      0.6709468439221382,
655      0.5821574740111828,
656      0.6687191724777222,
657      0.6399654373526573,
658      0.6364488303661346,
659      0.6354210749268532,
660      0.6799127534031868,
661      0.5936216339468956,
662      0.6423569396138191,
663      0.6152011044323444,
664      0.6492236107587814,
665      0.6289432123303413,
666      0.621874064207077,
667      0.6050795763731003,
668      0.6665552407503128,
669      0.59474778175354
670    ],
671    "confirmed": false
672  }
673}