1{
  2  "bench_version": 1,
  3  "track": "tabular",
  4  "model": "mlp_tiny",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "best_cfg": {
  9      "lr": 0.006,
 10      "weight_decay": 0.0001
 11    },
 12    "sweep": [
 13      {
 14        "cfg": {
 15          "lr": 0.0015,
 16          "weight_decay": 0.0
 17        },
 18        "mean": 6.582355260848999
 19      },
 20      {
 21        "cfg": {
 22          "lr": 0.0015,
 23          "weight_decay": 0.0001
 24        },
 25        "mean": 6.403406500816345
 26      },
 27      {
 28        "cfg": {
 29          "lr": 0.003,
 30          "weight_decay": 0.0
 31        },
 32        "mean": 6.050842046737671
 33      },
 34      {
 35        "cfg": {
 36          "lr": 0.003,
 37          "weight_decay": 0.0001
 38        },
 39        "mean": 5.60420548915863
 40      },
 41      {
 42        "cfg": {
 43          "lr": 0.006,
 44          "weight_decay": 0.0
 45        },
 46        "mean": 5.5363428592681885
 47      },
 48      {
 49        "cfg": {
 50          "lr": 0.006,
 51          "weight_decay": 0.0001
 52        },
 53        "mean": 4.698180794715881
 54      }
 55    ],
 56    "full": {
 57      "mean": 4.819164454936981,
 58      "std": 0.17166144964543392,
 59      "per_seed": [
 60        4.596839904785156,
 61        4.635165214538574,
 62        4.646710395812988,
 63        4.914007663726807,
 64        5.076730251312256,
 65        4.984879016876221,
 66        4.754410743713379,
 67        4.944572448730469
 68      ],
 69      "n": 8
 70    }
 71  },
 72  "idea": {
 73    "mean": 6.840818226337433,
 74    "std": 0.17257722313102455,
 75    "per_seed": [
 76      6.538304328918457,
 77      6.585258960723877,
 78      6.972299098968506,
 79      6.974996089935303,
 80      6.823678970336914,
 81      7.017618656158447,
 82      6.853421211242676,
 83      6.960968494415283
 84    ],
 85    "n": 8
 86  },
 87  "comparison": {
 88    "delta_mean": 2.0216537714004517,
 89    "idea_wins": 0,
 90    "n_pairs": 8,
 91    "per_seed_diffs": [
 92      1.9414644241333008,
 93      1.9500937461853027,
 94      2.3255887031555176,
 95      2.060988426208496,
 96      1.7469487190246582,
 97      2.0327396392822266,
 98      2.099010467529297,
 99      2.0163960456848145
100    ],
101    "p_value": 0.0081,
102    "mde": 0.13669161157539178,
103    "mde_rel_pct": 2.8364172431459234,
104    "verdict": "idea worse (significant)",
105    "system_worked": false
106  },
107  "mechanism_signature": {
108    "prediction": "nonzero replica circulation when k1!=k2",
109    "k1": 0.2,
110    "k2": 0.05,
111    "observed_epoch_area_seed0": -1.9684885011277033e-08,
112    "observed_mean_replica_separation": 0.07705571800470352,
113    "confirmed": true
114  },
115  "idea_sweep": [
116    {
117      "cfg": {
118        "lr": 0.0015,
119        "weight_decay": 0.0001
120      },
121      "result": {
122        "mean": 6.842114150524139,
123        "std": 0.19106420610526897,
124        "per_seed": [
125          6.48126745223999,
126          6.709563732147217,
127          7.120368957519531,
128          6.81691312789917,
129          6.813747406005859,
130          7.096852779388428,
131          6.863407611846924,
132          6.834792137145996
133        ],
134        "n": 8
135      }
136    },
137    {
138      "cfg": {
139        "lr": 0.003,
140        "weight_decay": 0.0001
141      },
142      "result": {
143        "mean": 6.840818226337433,
144        "std": 0.17257722313102455,
145        "per_seed": [
146          6.538304328918457,
147          6.585258960723877,
148          6.972299098968506,
149          6.974996089935303,
150          6.823678970336914,
151          7.017618656158447,
152          6.853421211242676,
153          6.960968494415283
154        ],
155        "n": 8
156      }
157    },
158    {
159      "cfg": {
160        "lr": 0.006,
161        "weight_decay": 0.0001
162      },
163      "result": {
164        "mean": 6.841454744338989,
165        "std": 0.2586476832475316,
166        "per_seed": [
167          7.215710163116455,
168          6.548867702484131,
169          7.098788261413574,
170          7.125370502471924,
171          6.484438896179199,
172          6.837231636047363,
173          6.688592910766602,
174          6.732637882232666
175        ],
176        "n": 8
177      }
178    }
179  ],
180  "protocol_notes": {
181    "epochs": 15,
182    "batch": 128,
183    "paired_seeds": [
184      0,
185      1,
186      2,
187      3,
188      4,
189      5,
190      6,
191      7
192    ],
193    "track_rationale": "tabular is the prescribed structural track for optimizer modifications",
194    "equal_budget_note": "same epochs and minibatch budget per replica; the idea uses two replicas and thus approximately 2x parameter-update compute"
195  }
196}