{ "bench_version": 1, "track": "tabular", "model": "mlp_tiny", "metric_direction": "lower is better", "n_seeds": 8, "baseline": { "best_cfg": { "lr": 0.006, "weight_decay": 0.0001 }, "sweep": [ { "cfg": { "lr": 0.0015, "weight_decay": 0.0 }, "mean": 12.669130086898804 }, { "cfg": { "lr": 0.0015, "weight_decay": 0.0001 }, "mean": 12.924439191818237 }, { "cfg": { "lr": 0.003, "weight_decay": 0.0 }, "mean": 9.31087851524353 }, { "cfg": { "lr": 0.003, "weight_decay": 0.0001 }, "mean": 9.370193243026733 }, { "cfg": { "lr": 0.006, "weight_decay": 0.0 }, "mean": 7.018180251121521 }, { "cfg": { "lr": 0.006, "weight_decay": 0.0001 }, "mean": 6.9977627992630005 } ], "full": { "mean": 7.143419861793518, "std": 0.27400662160821115, "per_seed": [ 6.931591987609863, 6.811943054199219, 7.096949577331543, 7.150566577911377, 6.830760955810547, 7.50044059753418, 7.220727920532227, 7.6043782234191895 ], "n": 8 } }, "idea": { "mean": 7.220555484294891, "std": 0.34912573865429086, "per_seed": [ 6.893190860748291, 7.041308879852295, 7.491337776184082, 7.315980434417725, 6.935866355895996, 7.914798259735107, 7.363597869873047, 6.808363437652588 ], "n": 8 }, "comparison": { "delta_mean": 0.07713562250137329, "idea_wins": 2, "n_pairs": 8, "per_seed_diffs": [ -0.038401126861572266, 0.22936582565307617, 0.39438819885253906, 0.16541385650634766, 0.10510540008544922, 0.41435766220092773, 0.1428699493408203, -0.7960147857666016 ], "p_value": 0.62745, "mde": 0.3201464002239144, "mde_rel_pct": 4.4816965321639985, "verdict": "no significant win", "system_worked": false }, "mechanism_signature": { "prediction": "warm-started exact paired pruning should produce nonincreasing active rank and balanced factor norms", "observed": { "0": { "seed": 0, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.500680896552979e-08, "pruned_columns": 0 }, "1": { "seed": 1, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.5005153313124484e-08, "pruned_columns": 0 }, "2": { "seed": 2, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.372116996779949e-08, "pruned_columns": 0 }, "3": { "seed": 3, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.407879780724064e-08, "pruned_columns": 0 }, "4": { "seed": 4, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.4962105480171883e-08, "pruned_columns": 0 }, "5": { "seed": 5, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 4.929386942582109e-08, "pruned_columns": 0 }, "6": { "seed": 6, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.0772395727705086e-08, "pruned_columns": 0 }, "7": { "seed": 7, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.675852322263431e-08, "pruned_columns": 0 } }, "predicted_monotone_rank": true, "predicted_balanced_gap": 0.0, "confirmed": true }, "idea_sweep": [ { "cfg": { "lr": 0.0015, "weight_decay": 0.0001, "lambda": 0.0 }, "result": { "mean": 13.893371224403381, "std": 2.523693976118532, "per_seed": [ 11.475872993469238, 11.684678077697754, 11.631744384765625, 16.454078674316406, 14.76885986328125, 18.05440902709961, 15.799757957458496, 11.277568817138672 ], "n": 8 }, "signature": { "0": { "seed": 0, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.464752540657653e-08, "pruned_columns": 0 }, "1": { "seed": 1, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 4.9146513401154834e-08, "pruned_columns": 0 }, "2": { "seed": 2, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.0755838862196845e-08, "pruned_columns": 0 }, "3": { "seed": 3, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 4.9900677821944504e-08, "pruned_columns": 0 }, "4": { "seed": 4, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.239910578719926e-08, "pruned_columns": 0 }, "5": { "seed": 5, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 4.998594550818956e-08, "pruned_columns": 0 }, "6": { "seed": 6, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 4.5999057067073563e-08, "pruned_columns": 0 }, "7": { "seed": 7, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.2653253316887254e-08, "pruned_columns": 0 } } }, { "cfg": { "lr": 0.0015, "weight_decay": 0.0001, "lambda": 1e-06 }, "result": { "mean": 13.893371224403381, "std": 2.523693976118532, "per_seed": [ 11.475872993469238, 11.684678077697754, 11.631744384765625, 16.454078674316406, 14.76885986328125, 18.05440902709961, 15.799757957458496, 11.277568817138672 ], "n": 8 }, "signature": { "0": { "seed": 0, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.464752540657653e-08, "pruned_columns": 0 }, "1": { "seed": 1, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 4.9146513401154834e-08, "pruned_columns": 0 }, "2": { "seed": 2, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.0755838862196845e-08, "pruned_columns": 0 }, "3": { "seed": 3, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 4.9900677821944504e-08, "pruned_columns": 0 }, "4": { "seed": 4, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.239910578719926e-08, "pruned_columns": 0 }, "5": { "seed": 5, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 4.998594550818956e-08, "pruned_columns": 0 }, "6": { "seed": 6, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 4.5999057067073563e-08, "pruned_columns": 0 }, "7": { "seed": 7, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.2653253316887254e-08, "pruned_columns": 0 } } }, { "cfg": { "lr": 0.0015, "weight_decay": 0.0001, "lambda": 3e-06 }, "result": { "mean": 13.893371224403381, "std": 2.523693976118532, "per_seed": [ 11.475872993469238, 11.684678077697754, 11.631744384765625, 16.454078674316406, 14.76885986328125, 18.05440902709961, 15.799757957458496, 11.277568817138672 ], "n": 8 }, "signature": { "0": { "seed": 0, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.464752540657653e-08, "pruned_columns": 0 }, "1": { "seed": 1, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 4.9146513401154834e-08, "pruned_columns": 0 }, "2": { "seed": 2, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.0755838862196845e-08, "pruned_columns": 0 }, "3": { "seed": 3, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 4.9900677821944504e-08, "pruned_columns": 0 }, "4": { "seed": 4, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.239910578719926e-08, "pruned_columns": 0 }, "5": { "seed": 5, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 4.998594550818956e-08, "pruned_columns": 0 }, "6": { "seed": 6, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 4.5999057067073563e-08, "pruned_columns": 0 }, "7": { "seed": 7, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.2653253316887254e-08, "pruned_columns": 0 } } }, { "cfg": { "lr": 0.003, "weight_decay": 0.0001, "lambda": 0.0 }, "result": { "mean": 9.69086343050003, "std": 1.88049376973306, "per_seed": [ 7.3153910636901855, 9.046968460083008, 8.134467124938965, 12.36645221710205, 8.763382911682129, 12.712215423583984, 10.7437744140625, 8.444255828857422 ], "n": 8 }, "signature": { "0": { "seed": 0, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.5834651306380725e-08, "pruned_columns": 0 }, "1": { "seed": 1, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.253073270792027e-08, "pruned_columns": 0 }, "2": { "seed": 2, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.187177028551402e-08, "pruned_columns": 0 }, "3": { "seed": 3, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.220870205688596e-08, "pruned_columns": 0 }, "4": { "seed": 4, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.177325697343823e-08, "pruned_columns": 0 }, "5": { "seed": 5, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 4.8814548660026786e-08, "pruned_columns": 0 }, "6": { "seed": 6, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 4.899832962578109e-08, "pruned_columns": 0 }, "7": { "seed": 7, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.3645836288751523e-08, "pruned_columns": 0 } } }, { "cfg": { "lr": 0.003, "weight_decay": 0.0001, "lambda": 1e-06 }, "result": { "mean": 9.69086343050003, "std": 1.88049376973306, "per_seed": [ 7.3153910636901855, 9.046968460083008, 8.134467124938965, 12.36645221710205, 8.763382911682129, 12.712215423583984, 10.7437744140625, 8.444255828857422 ], "n": 8 }, "signature": { "0": { "seed": 0, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.5834651306380725e-08, "pruned_columns": 0 }, "1": { "seed": 1, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.253073270792027e-08, "pruned_columns": 0 }, "2": { "seed": 2, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.187177028551402e-08, "pruned_columns": 0 }, "3": { "seed": 3, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.220870205688596e-08, "pruned_columns": 0 }, "4": { "seed": 4, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.177325697343823e-08, "pruned_columns": 0 }, "5": { "seed": 5, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 4.8814548660026786e-08, "pruned_columns": 0 }, "6": { "seed": 6, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 4.899832962578109e-08, "pruned_columns": 0 }, "7": { "seed": 7, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.3645836288751523e-08, "pruned_columns": 0 } } }, { "cfg": { "lr": 0.003, "weight_decay": 0.0001, "lambda": 3e-06 }, "result": { "mean": 9.69086343050003, "std": 1.88049376973306, "per_seed": [ 7.3153910636901855, 9.046968460083008, 8.134467124938965, 12.36645221710205, 8.763382911682129, 12.712215423583984, 10.7437744140625, 8.444255828857422 ], "n": 8 }, "signature": { "0": { "seed": 0, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.5834651306380725e-08, "pruned_columns": 0 }, "1": { "seed": 1, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.253073270792027e-08, "pruned_columns": 0 }, "2": { "seed": 2, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.187177028551402e-08, "pruned_columns": 0 }, "3": { "seed": 3, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.220870205688596e-08, "pruned_columns": 0 }, "4": { "seed": 4, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.177325697343823e-08, "pruned_columns": 0 }, "5": { "seed": 5, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 4.8814548660026786e-08, "pruned_columns": 0 }, "6": { "seed": 6, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 4.899832962578109e-08, "pruned_columns": 0 }, "7": { "seed": 7, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.3645836288751523e-08, "pruned_columns": 0 } } }, { "cfg": { "lr": 0.006, "weight_decay": 0.0001, "lambda": 0.0 }, "result": { "mean": 7.220555484294891, "std": 0.34912573865429086, "per_seed": [ 6.893190860748291, 7.041308879852295, 7.491337776184082, 7.315980434417725, 6.935866355895996, 7.914798259735107, 7.363597869873047, 6.808363437652588 ], "n": 8 }, "signature": { "0": { "seed": 0, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.500680896552979e-08, "pruned_columns": 0 }, "1": { "seed": 1, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.5005153313124484e-08, "pruned_columns": 0 }, "2": { "seed": 2, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.372116996779949e-08, "pruned_columns": 0 }, "3": { "seed": 3, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.407879780724064e-08, "pruned_columns": 0 }, "4": { "seed": 4, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.4962105480171883e-08, "pruned_columns": 0 }, "5": { "seed": 5, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 4.929386942582109e-08, "pruned_columns": 0 }, "6": { "seed": 6, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.0772395727705086e-08, "pruned_columns": 0 }, "7": { "seed": 7, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.675852322263431e-08, "pruned_columns": 0 } } }, { "cfg": { "lr": 0.006, "weight_decay": 0.0001, "lambda": 1e-06 }, "result": { "mean": 7.220555484294891, "std": 0.34912573865429086, "per_seed": [ 6.893190860748291, 7.041308879852295, 7.491337776184082, 7.315980434417725, 6.935866355895996, 7.914798259735107, 7.363597869873047, 6.808363437652588 ], "n": 8 }, "signature": { "0": { "seed": 0, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.500680896552979e-08, "pruned_columns": 0 }, "1": { "seed": 1, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.5005153313124484e-08, "pruned_columns": 0 }, "2": { "seed": 2, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.372116996779949e-08, "pruned_columns": 0 }, "3": { "seed": 3, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.407879780724064e-08, "pruned_columns": 0 }, "4": { "seed": 4, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.4962105480171883e-08, "pruned_columns": 0 }, "5": { "seed": 5, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 4.929386942582109e-08, "pruned_columns": 0 }, "6": { "seed": 6, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.0772395727705086e-08, "pruned_columns": 0 }, "7": { "seed": 7, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.675852322263431e-08, "pruned_columns": 0 } } }, { "cfg": { "lr": 0.006, "weight_decay": 0.0001, "lambda": 3e-06 }, "result": { "mean": 7.220555484294891, "std": 0.34912573865429086, "per_seed": [ 6.893190860748291, 7.041308879852295, 7.491337776184082, 7.315980434417725, 6.935866355895996, 7.914798259735107, 7.363597869873047, 6.808363437652588 ], "n": 8 }, "signature": { "0": { "seed": 0, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.500680896552979e-08, "pruned_columns": 0 }, "1": { "seed": 1, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.5005153313124484e-08, "pruned_columns": 0 }, "2": { "seed": 2, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.372116996779949e-08, "pruned_columns": 0 }, "3": { "seed": 3, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.407879780724064e-08, "pruned_columns": 0 }, "4": { "seed": 4, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.4962105480171883e-08, "pruned_columns": 0 }, "5": { "seed": 5, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 4.929386942582109e-08, "pruned_columns": 0 }, "6": { "seed": 6, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.0772395727705086e-08, "pruned_columns": 0 }, "7": { "seed": 7, "ranks_by_lambda_stage": [ 43, 43, 43, 43 ], "observed_monotone": true, "mean_balanced_norm_gap": 5.675852322263431e-08, "pruned_columns": 0 } } } ], "runtime_sec": 202.02961421012878, "device": "cuda" }