Probe-Then-Partitioned Multi-Task Trunk / bench_report.json

Failed on benchmark

Raw ⬇ ZIP
  1{
  2  "bench_version": 1,
  3  "track": "correlated_multitask_regression",
  4  "model": "mlp_tiny",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "best_cfg": {
  9      "lr": 0.003
 10    },
 11    "sweep": [
 12      {
 13        "cfg": {
 14          "lr": 0.001
 15        },
 16        "mean": 0.4304336408774058
 17      },
 18      {
 19        "cfg": {
 20          "lr": 0.003
 21        },
 22        "mean": 0.4079873462518056
 23      },
 24      {
 25        "cfg": {
 26          "lr": 0.01
 27        },
 28        "mean": 0.41518495480219525
 29      }
 30    ],
 31    "full": {
 32      "mean": 0.38796478137373924,
 33      "std": 0.031118597630805808,
 34      "per_seed": [
 35        0.3507640063762665,
 36        0.3547458350658417,
 37        0.3831292986869812,
 38        0.3947278559207916,
 39        0.43335363268852234,
 40        0.38597869873046875,
 41        0.43800339102745056,
 42        0.3630155324935913
 43      ],
 44      "n": 8
 45    }
 46  },
 47  "idea": {
 48    "mean": 0.39804549887776375,
 49    "std": 0.018242689312241732,
 50    "per_seed": [
 51      0.39564016461372375,
 52      0.43557506799697876,
 53      0.39274677634239197,
 54      0.3693575859069824,
 55      0.4083901047706604,
 56      0.39621248841285706,
 57      0.38226383924484253,
 58      0.4041779637336731
 59    ],
 60    "n": 8
 61  },
 62  "comparison": {
 63    "delta_mean": 0.010080717504024506,
 64    "idea_wins": 3,
 65    "n_pairs": 8,
 66    "per_seed_diffs": [
 67      0.044876158237457275,
 68      0.08082923293113708,
 69      0.009617477655410767,
 70      -0.025370270013809204,
 71      -0.02496352791786194,
 72      0.010233789682388306,
 73      -0.05573955178260803,
 74      0.04116243124008179
 75    ],
 76    "p_value": 0.54355,
 77    "mde": 0.037393353165640304,
 78    "mde_rel_pct": 9.638337024622361,
 79    "verdict": "no significant win",
 80    "system_worked": false
 81  },
 82  "mechanism_signature": {
 83    "track_rationale": "multi-task regression with six task heads and latent task groups; custom track is structurally matched",
 84    "idea_grid": {
 85      "0.001": {
 86        "mean": 0.4304336408774058,
 87        "std": 0.021430113433642317,
 88        "per_seed": [
 89          0.4190196692943573,
 90          0.460454523563385,
 91          0.4118267297744751
 92        ],
 93        "n": 3
 94      },
 95      "0.003": {
 96        "mean": 0.4079873363176982,
 97        "std": 0.01954320219971055,
 98        "per_seed": [
 99          0.39564016461372375,
100          0.43557506799697876,
101          0.39274677634239197
102        ],
103        "n": 3
104      },
105      "0.01": {
106        "mean": 0.41518494486808777,
107        "std": 0.01594100724609616,
108        "per_seed": [
109          0.40816590189933777,
110          0.43724772334098816,
111          0.4001412093639374
112        ],
113        "n": 3
114      }
115    },
116    "selected_idea_lr": 0.003,
117    "probe_epochs": 4,
118    "groups_seed0": [
119      0,
120      0,
121      0,
122      0,
123      0,
124      0
125    ],
126    "mechanism_signature": {
127      "predicted": "partitioning reduces negative cross-task gradient cosine",
128      "baseline_conflict_rate": 0.26666666666666666,
129      "idea_conflict_rate": 0.26666666666666666,
130      "observed_reduction": 0.0,
131      "confirmed": false
132    }
133  },
134  "parameter_note": "Partitioned trunks replicate the trunk per discovered cluster; heads remain separate. Equal optimizer/data/epoch budget, but inference parameter count can differ."
135}