{
  "bench_version": 1,
  "track": "tabular",
  "model": "mlp_tiny",
  "metric_direction": "lower is better",
  "n_seeds": 8,
  "baseline": {
    "best_cfg": {
      "lr": 0.03
    },
    "sweep": [
      {
        "cfg": {
          "lr": 0.001
        },
        "mean": 24.142436981201172
      },
      {
        "cfg": {
          "lr": 0.003
        },
        "mean": 13.75401242574056
      },
      {
        "cfg": {
          "lr": 0.01
        },
        "mean": 8.325917879740397
      },
      {
        "cfg": {
          "lr": 0.03
        },
        "mean": 6.865411599477132
      }
    ],
    "full": {
      "mean": 7.418325066566467,
      "std": 0.9955428790176792,
      "per_seed": [
        6.716945648193359,
        5.94451379776001,
        7.934775352478027,
        9.200451850891113,
        7.416287899017334,
        6.373697280883789,
        8.270235061645508,
        7.489693641662598
      ],
      "n": 8
    }
  },
  "idea": {
    "mean": 124.02495861053467,
    "std": 19.888619557248344,
    "per_seed": [
      133.7542266845703,
      95.59589385986328,
      161.67819213867188,
      123.55552673339844,
      96.50812530517578,
      130.4013214111328,
      120.75763702392578,
      129.94874572753906
    ],
    "n": 8,
    "chosen_cfg": {
      "lr": 0.03
    },
    "trials": [
      {
        "cfg": {
          "lr": 0.003
        },
        "mean": 235.4395955403646
      },
      {
        "cfg": {
          "lr": 0.01
        },
        "mean": 223.33053588867188
      },
      {
        "cfg": {
          "lr": 0.03
        },
        "mean": 130.3427708943685
      }
    ]
  },
  "comparison": {
    "delta_mean": 116.6066335439682,
    "idea_wins": 0,
    "n_pairs": 8,
    "per_seed_diffs": [
      127.03728103637695,
      89.65138006210327,
      153.74341678619385,
      114.35507488250732,
      89.09183740615845,
      124.02762413024902,
      112.48740196228027,
      122.45905208587646
    ],
    "p_value": 0.0081,
    "mde": 17.547701974609687,
    "mde_rel_pct": 236.54533627402162,
    "verdict": "idea worse (significant)",
    "system_worked": false
  },
  "mechanism_signature": {
    "predicted": {
      "transport_error": 0.0,
      "call_reduction_fraction": 0.5
    },
    "observed": {
      "mean_transport_error": 1.59325485071804e-06,
      "mean_idea_grad_calls": 73.0,
      "mean_baseline_grad_calls": 72.0,
      "call_reduction_fraction": -0.013888888888888888
    },
    "confirmed": false
  }
}