1{
  2  "bench_version": 1,
  3  "track": "tabular",
  4  "model": "masked_mlp_tiny",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "best_cfg": {
  9      "lr": 0.01,
 10      "epochs": 4,
 11      "mask_count": 20
 12    },
 13    "sweep": [
 14      {
 15        "cfg": {
 16          "lr": 0.001,
 17          "epochs": 4,
 18          "mask_count": 20
 19        },
 20        "mean": 233.01842498779297
 21      },
 22      {
 23        "cfg": {
 24          "lr": 0.003,
 25          "epochs": 4,
 26          "mask_count": 20
 27        },
 28        "mean": 228.20503997802734
 29      },
 30      {
 31        "cfg": {
 32          "lr": 0.01,
 33          "epochs": 4,
 34          "mask_count": 20
 35        },
 36        "mean": 191.4046401977539
 37      }
 38    ],
 39    "full": {
 40      "mean": 185.9184112548828,
 41      "std": 13.31649292644178,
 42      "per_seed": [
 43        211.56298828125,
 44        186.1365509033203,
 45        195.3025665283203,
 46        172.616455078125,
 47        170.3512420654297,
 48        193.65509033203125,
 49        172.0006561279297,
 50        185.72174072265625
 51      ],
 52      "n": 8
 53    }
 54  },
 55  "idea": {
 56    "mean": 208.20309448242188,
 57    "std": 13.805027643998322,
 58    "per_seed": [
 59      226.8892364501953,
 60      212.61376953125,
 61      219.84451293945312,
 62      193.2020263671875,
 63      220.1836700439453,
 64      202.975341796875,
 65      183.25924682617188,
 66      206.65695190429688
 67    ],
 68    "n": 8
 69  },
 70  "comparison": {
 71    "delta_mean": 22.284683227539062,
 72    "idea_wins": 0,
 73    "n_pairs": 8,
 74    "per_seed_diffs": [
 75      15.326248168945312,
 76      26.477218627929688,
 77      24.541946411132812,
 78      20.5855712890625,
 79      49.832427978515625,
 80      9.32025146484375,
 81      11.258590698242188,
 82      20.935211181640625
 83    ],
 84    "p_value": 0.0081,
 85    "mde": 10.597026350103498,
 86    "mde_rel_pct": 5.699826218703871,
 87    "verdict": "idea worse (significant)",
 88    "system_worked": false
 89  },
 90  "idea_sweep": [
 91    {
 92      "cfg": {
 93        "lr": 0.001,
 94        "epochs": 4,
 95        "mask_count": 1,
 96        "canonical": true
 97      },
 98      "mean": 232.431640625
 99    },
100    {
101      "cfg": {
102        "lr": 0.003,
103        "epochs": 4,
104        "mask_count": 1,
105        "canonical": true
106      },
107      "mean": 228.43239402770996
108    },
109    {
110      "cfg": {
111        "lr": 0.01,
112        "epochs": 4,
113        "mask_count": 1,
114        "canonical": true
115      },
116      "mean": 208.20309448242188
117    }
118  ],
119  "runtime_sec": 36.57714536000003,
120  "track_justification": "Tabular is the built-in architecture/regularization-adjacent track; the experiment directly operates on exchangeable hidden channels and structured pruning masks.",
121  "mask_definition": "retain k=3 of six first-layer hidden channels; exact S_6 symmetry represented by channel permutations.",
122  "mechanism_signature": {
123    "predicted_orbits": 1,
124    "observed_candidate_masks": 20,
125    "predicted_evaluation_reduction": 20,
126    "observed_evaluation_reduction": 20,
127    "trained_model_mask_spread": [
128      {
129        "seed": 0,
130        "mask_metric_std": 18.708847715759166,
131        "mask_metric_range": 73.77125549316406
132      },
133      {
134        "seed": 1,
135        "mask_metric_std": 11.349517459757626,
136        "mask_metric_range": 53.40818786621094
137      },
138      {
139        "seed": 2,
140        "mask_metric_std": 13.9794193204962,
141        "mask_metric_range": 43.052520751953125
142      },
143      {
144        "seed": 3,
145        "mask_metric_std": 9.279172900445374,
146        "mask_metric_range": 36.04695129394531
147      },
148      {
149        "seed": 4,
150        "mask_metric_std": 14.554493291709218,
151        "mask_metric_range": 54.18971252441406
152      },
153      {
154        "seed": 5,
155        "mask_metric_std": 12.235878564511857,
156        "mask_metric_range": 44.35304260253906
157      },
158      {
159        "seed": 6,
160        "mask_metric_std": 12.367370327709848,
161        "mask_metric_range": 58.37591552734375
162      },
163      {
164        "seed": 7,
165        "mask_metric_std": 18.78608365872588,
166        "mask_metric_range": 67.00553894042969
167      }
168    ],
169    "observed_nonzero_spread_seeds": 8,
170    "confirmed": false,
171    "note": "The exact reduction is structural; finite independently trained masked models can differ because optimization noise breaks numerical equality."
172  }
173}