1{
  2  "bench_version": 1,
  3  "track": "budgeted_route_costs",
  4  "model": "mlp_tiny",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "best_cfg": {
  9      "lr": 0.01,
 10      "wd": 0.0001
 11    },
 12    "sweep": [
 13      {
 14        "cfg": {
 15          "lr": 0.001,
 16          "wd": 0.0
 17        },
 18        "mean": 0.024585966020822525
 19      },
 20      {
 21        "cfg": {
 22          "lr": 0.001,
 23          "wd": 0.0001
 24        },
 25        "mean": 0.02389408415183425
 26      },
 27      {
 28        "cfg": {
 29          "lr": 0.003,
 30          "wd": 0.0
 31        },
 32        "mean": 0.004350764560513198
 33      },
 34      {
 35        "cfg": {
 36          "lr": 0.003,
 37          "wd": 0.0001
 38        },
 39        "mean": 0.004227392724715173
 40      },
 41      {
 42        "cfg": {
 43          "lr": 0.01,
 44          "wd": 0.0
 45        },
 46        "mean": 0.002535940962843597
 47      },
 48      {
 49        "cfg": {
 50          "lr": 0.01,
 51          "wd": 0.0001
 52        },
 53        "mean": 0.0022327409242279828
 54      }
 55    ],
 56    "full": {
 57      "mean": 0.0023824743693694472,
 58      "std": 0.0003298540492088312,
 59      "per_seed": [
 60        0.002391217043623328,
 61        0.0022307508625090122,
 62        0.0022920493502169847,
 63        0.002016946440562606,
 64        0.0019817666616290808,
 65        0.003033609827980399,
 66        0.0027303267270326614,
 67        0.0023831280414015055
 68      ],
 69      "n": 8
 70    }
 71  },
 72  "idea": {
 73    "mean": 0.12442152807489038,
 74    "std": 0.06743830819346265,
 75    "per_seed": [
 76      0.09885232150554657,
 77      0.07384644448757172,
 78      0.14970475435256958,
 79      0.25081515312194824,
 80      0.05385749414563179,
 81      0.21027871966362,
 82      0.08540374785661697,
 83      0.07261358946561813
 84    ],
 85    "n": 8
 86  },
 87  "comparison": {
 88    "delta_mean": 0.12203905370552093,
 89    "idea_wins": 0,
 90    "n_pairs": 8,
 91    "per_seed_diffs": [
 92      0.09646110446192324,
 93      0.0716156936250627,
 94      0.1474127050023526,
 95      0.24879820668138564,
 96      0.05187572748400271,
 97      0.2072451098356396,
 98      0.08267342112958431,
 99      0.07023046142421663
100    ],
101    "p_value": 0.0081,
102    "mde": 0.06023392098974088,
103    "mde_rel_pct": 2528.208561827365,
104    "verdict": "idea worse (significant)",
105    "system_worked": false
106  },
107  "custom_track": {
108    "name": "budgeted_route_costs",
109    "file": "route_track.py",
110    "domain": "graph-decision"
111  },
112  "idea_sweep": [
113    {
114      "cfg": {
115        "lr": 0.01,
116        "wd": 0.0001
117      },
118      "result": {
119        "mean": 0.12442152807489038,
120        "std": 0.06743830819346265,
121        "per_seed": [
122          0.09885232150554657,
123          0.07384644448757172,
124          0.14970475435256958,
125          0.25081515312194824,
126          0.05385749414563179,
127          0.21027871966362,
128          0.08540374785661697,
129          0.07261358946561813
130        ],
131        "n": 8
132      }
133    },
134    {
135      "cfg": {
136        "lr": 0.001,
137        "wd": 0.0001
138      },
139      "result": {
140        "mean": 0.27160821482539177,
141        "std": 0.05016213483864423,
142        "per_seed": [
143          0.3274783790111542,
144          0.2356351912021637,
145          0.31766167283058167,
146          0.16266748309135437,
147          0.25924399495124817,
148          0.2929310202598572,
149          0.2722110152244568,
150          0.3050369620323181
151        ],
152        "n": 8
153      }
154    },
155    {
156      "cfg": {
157        "lr": 0.01,
158        "wd": 0.0001
159      },
160      "result": {
161        "mean": 0.12442152807489038,
162        "std": 0.06743830819346265,
163        "per_seed": [
164          0.09885232150554657,
165          0.07384644448757172,
166          0.14970475435256958,
167          0.25081515312194824,
168          0.05385749414563179,
169          0.21027871966362,
170          0.08540374785661697,
171          0.07261358946561813
172        ],
173        "n": 8
174      }
175    }
176  ],
177  "mechanism_signature": {
178    "baseline_worst_regret_mean": 0.006290760776028037,
179    "idea_worst_regret_mean": 0.006380649516358972,
180    "predicted_effect": "adversarial training reduces budgeted path flips/regret",
181    "confirmed": false
182  }
183}