1{
  2  "bench_version": 1,
  3  "track": "stochastic_intervention_scm",
  4  "model": "mlp_tiny",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "best_cfg": {
  9      "lr": 0.003
 10    },
 11    "sweep": [
 12      {
 13        "cfg": {
 14          "lr": 0.003
 15        },
 16        "mean": 2.497592270374298
 17      },
 18      {
 19        "cfg": {
 20          "lr": 0.01
 21        },
 22        "mean": 2.5648475885391235
 23      },
 24      {
 25        "cfg": {
 26          "lr": 0.03
 27        },
 28        "mean": 2.6781041622161865
 29      }
 30    ],
 31    "full": {
 32      "mean": 2.414719134569168,
 33      "std": 0.23689432610640407,
 34      "per_seed": [
 35        2.342486619949341,
 36        2.4945199489593506,
 37        2.7365739345550537,
 38        2.4167885780334473,
 39        2.2174108028411865,
 40        2.058196783065796,
 41        2.258479356765747,
 42        2.793297052383423
 43      ],
 44      "n": 8
 45    }
 46  },
 47  "idea": {
 48    "mean": 2.2138066589832306,
 49    "std": 0.5331661449712295,
 50    "per_seed": [
 51      2.4092166423797607,
 52      2.4376542568206787,
 53      2.0347108840942383,
 54      1.5852525234222412,
 55      1.881182074546814,
 56      1.8407953977584839,
 57      3.4355263710021973,
 58      2.0861151218414307
 59    ],
 60    "n": 8,
 61    "best_cfg": {
 62      "lr": 0.03
 63    },
 64    "sweep": [
 65      {
 66        "cfg": {
 67          "lr": 0.003
 68        },
 69        "mean": 7.919645547866821,
 70        "std": 2.5149207407136784,
 71        "per_seed": [
 72          10.004054069519043,
 73          5.846197605133057,
 74          7.366078853607178,
 75          4.32836389541626,
 76          7.836812496185303,
 77          5.4305877685546875,
 78          11.452704429626465,
 79          11.092365264892578
 80        ],
 81        "n": 8
 82      },
 83      {
 84        "cfg": {
 85          "lr": 0.01
 86        },
 87        "mean": 3.1929454803466797,
 88        "std": 0.8334051410010931,
 89        "per_seed": [
 90          3.184743642807007,
 91          3.0707268714904785,
 92          3.08829665184021,
 93          2.054521322250366,
 94          2.8313469886779785,
 95          2.428166151046753,
 96          4.886856555938721,
 97          3.998905658721924
 98        ],
 99        "n": 8
100      },
101      {
102        "cfg": {
103          "lr": 0.03
104        },
105        "mean": 2.2138066589832306,
106        "std": 0.5331661449712295,
107        "per_seed": [
108          2.4092166423797607,
109          2.4376542568206787,
110          2.0347108840942383,
111          1.5852525234222412,
112          1.881182074546814,
113          1.8407953977584839,
114          3.4355263710021973,
115          2.0861151218414307
116        ],
117        "n": 8
118      }
119    ]
120  },
121  "comparison": {
122    "delta_mean": -0.2009124755859375,
123    "idea_wins": 6,
124    "n_pairs": 8,
125    "per_seed_diffs": [
126      0.06673002243041992,
127      -0.056865692138671875,
128      -0.7018630504608154,
129      -0.831536054611206,
130      -0.33622872829437256,
131      -0.217401385307312,
132      1.1770470142364502,
133      -0.7071819305419922
134    ],
135    "p_value": 0.41835,
136    "mde": 0.5395513694218982,
137    "mde_rel_pct": 22.34427025891624,
138    "verdict": "no significant win",
139    "system_worked": false
140  },
141  "custom_track": {
142    "name": "stochastic_intervention_scm",
143    "file": "custom_stochastic_intervention_track.py",
144    "domain": "causal-world-model"
145  },
146  "mechanism_signature": {
147    "baseline_predicted_selected_mean": 1.6279271692037582,
148    "idea_predicted_selected_mean": 1.8218559473752975,
149    "observed_selected_mean": 1.7572368532419205,
150    "baseline_abs_error": 0.12930968403816223,
151    "idea_abs_error": 0.06461909413337708,
152    "confirmed": true
153  }
154}