{
  "bench_version": 1,
  "track": "stochastic_intervention_scm",
  "model": "mlp_tiny",
  "metric_direction": "lower is better",
  "n_seeds": 8,
  "baseline": {
    "best_cfg": {
      "lr": 0.003
    },
    "sweep": [
      {
        "cfg": {
          "lr": 0.003
        },
        "mean": 2.497592270374298
      },
      {
        "cfg": {
          "lr": 0.01
        },
        "mean": 2.5648475885391235
      },
      {
        "cfg": {
          "lr": 0.03
        },
        "mean": 2.6781041622161865
      }
    ],
    "full": {
      "mean": 2.414719134569168,
      "std": 0.23689432610640407,
      "per_seed": [
        2.342486619949341,
        2.4945199489593506,
        2.7365739345550537,
        2.4167885780334473,
        2.2174108028411865,
        2.058196783065796,
        2.258479356765747,
        2.793297052383423
      ],
      "n": 8
    }
  },
  "idea": {
    "mean": 2.2138066589832306,
    "std": 0.5331661449712295,
    "per_seed": [
      2.4092166423797607,
      2.4376542568206787,
      2.0347108840942383,
      1.5852525234222412,
      1.881182074546814,
      1.8407953977584839,
      3.4355263710021973,
      2.0861151218414307
    ],
    "n": 8,
    "best_cfg": {
      "lr": 0.03
    },
    "sweep": [
      {
        "cfg": {
          "lr": 0.003
        },
        "mean": 7.919645547866821,
        "std": 2.5149207407136784,
        "per_seed": [
          10.004054069519043,
          5.846197605133057,
          7.366078853607178,
          4.32836389541626,
          7.836812496185303,
          5.4305877685546875,
          11.452704429626465,
          11.092365264892578
        ],
        "n": 8
      },
      {
        "cfg": {
          "lr": 0.01
        },
        "mean": 3.1929454803466797,
        "std": 0.8334051410010931,
        "per_seed": [
          3.184743642807007,
          3.0707268714904785,
          3.08829665184021,
          2.054521322250366,
          2.8313469886779785,
          2.428166151046753,
          4.886856555938721,
          3.998905658721924
        ],
        "n": 8
      },
      {
        "cfg": {
          "lr": 0.03
        },
        "mean": 2.2138066589832306,
        "std": 0.5331661449712295,
        "per_seed": [
          2.4092166423797607,
          2.4376542568206787,
          2.0347108840942383,
          1.5852525234222412,
          1.881182074546814,
          1.8407953977584839,
          3.4355263710021973,
          2.0861151218414307
        ],
        "n": 8
      }
    ]
  },
  "comparison": {
    "delta_mean": -0.2009124755859375,
    "idea_wins": 6,
    "n_pairs": 8,
    "per_seed_diffs": [
      0.06673002243041992,
      -0.056865692138671875,
      -0.7018630504608154,
      -0.831536054611206,
      -0.33622872829437256,
      -0.217401385307312,
      1.1770470142364502,
      -0.7071819305419922
    ],
    "p_value": 0.41835,
    "mde": 0.5395513694218982,
    "mde_rel_pct": 22.34427025891624,
    "verdict": "no significant win",
    "system_worked": false
  },
  "custom_track": {
    "name": "stochastic_intervention_scm",
    "file": "custom_stochastic_intervention_track.py",
    "domain": "causal-world-model"
  },
  "mechanism_signature": {
    "baseline_predicted_selected_mean": 1.6279271692037582,
    "idea_predicted_selected_mean": 1.8218559473752975,
    "observed_selected_mean": 1.7572368532419205,
    "baseline_abs_error": 0.12930968403816223,
    "idea_abs_error": 0.06461909413337708,
    "confirmed": true
  }
}