{
  "bench_version": 1,
  "track": "unordered_pointset_denoising",
  "model": "mlp_med",
  "metric_direction": "lower is better",
  "n_seeds": 8,
  "baseline": {
    "best_cfg": {
      "lr": 0.006,
      "weight_decay": 0.0
    },
    "sweep": [
      {
        "cfg": {
          "lr": 0.001,
          "weight_decay": 0.0
        },
        "mean": 0.04259627405554056
      },
      {
        "cfg": {
          "lr": 0.001,
          "weight_decay": 0.0001
        },
        "mean": 0.04272811021655798
      },
      {
        "cfg": {
          "lr": 0.003,
          "weight_decay": 0.0
        },
        "mean": 0.03448029048740864
      },
      {
        "cfg": {
          "lr": 0.003,
          "weight_decay": 0.0001
        },
        "mean": 0.034148878417909145
      },
      {
        "cfg": {
          "lr": 0.006,
          "weight_decay": 0.0
        },
        "mean": 0.03395335283130407
      },
      {
        "cfg": {
          "lr": 0.006,
          "weight_decay": 0.0001
        },
        "mean": 0.034376708790659904
      }
    ],
    "full": {
      "mean": 0.034991197753697634,
      "std": 0.002117625587373799,
      "per_seed": [
        0.036085259169340134,
        0.03398258239030838,
        0.03428776562213898,
        0.0314578041434288,
        0.034709829837083817,
        0.03937392309308052,
        0.035850487649440765,
        0.034181930124759674
      ],
      "n": 8
    }
  },
  "idea": {
    "mean": 0.0419774423353374,
    "std": 0.0049312628769484414,
    "per_seed": [
      0.039885085076093674,
      0.03695981204509735,
      0.038052886724472046,
      0.04973628371953964,
      0.04566482454538345,
      0.036613475531339645,
      0.04028519243001938,
      0.04862197861075401
    ],
    "n": 8
  },
  "comparison": {
    "delta_mean": 0.006986244581639767,
    "idea_wins": 1,
    "n_pairs": 8,
    "per_seed_diffs": [
      0.00379982590675354,
      0.002977229654788971,
      0.003765121102333069,
      0.01827847957611084,
      0.010954994708299637,
      -0.0027604475617408752,
      0.004434704780578613,
      0.014440048485994339
    ],
    "p_value": 0.0163,
    "mde": 0.005801091064441157,
    "mde_rel_pct": 16.578715325136695,
    "verdict": "idea worse (significant)",
    "system_worked": false
  },
  "custom_track": {
    "name": "unordered_pointset_denoising",
    "file": "bench/custom_tracks/unordered_pointset_denoising.py",
    "domain": "point-set-diffusion"
  },
  "baseline_grid": [
    {
      "lr": 0.001,
      "weight_decay": 0.0
    },
    {
      "lr": 0.001,
      "weight_decay": 0.0001
    },
    {
      "lr": 0.003,
      "weight_decay": 0.0
    },
    {
      "lr": 0.003,
      "weight_decay": 0.0001
    },
    {
      "lr": 0.006,
      "weight_decay": 0.0
    },
    {
      "lr": 0.006,
      "weight_decay": 0.0001
    }
  ],
  "idea_grid": [
    {
      "lr": 0.001,
      "weight_decay": 0.0,
      "tree_lambda": 0.02
    },
    {
      "lr": 0.003,
      "weight_decay": 0.0,
      "tree_lambda": 0.05
    },
    {
      "lr": 0.006,
      "weight_decay": 0.0001,
      "tree_lambda": 0.1
    }
  ],
  "idea_trials": [
    {
      "cfg": {
        "lr": 0.001,
        "weight_decay": 0.0,
        "tree_lambda": 0.02
      },
      "mean": 0.04521314054727554
    },
    {
      "cfg": {
        "lr": 0.003,
        "weight_decay": 0.0,
        "tree_lambda": 0.05
      },
      "mean": 0.04115851689130068
    },
    {
      "cfg": {
        "lr": 0.006,
        "weight_decay": 0.0001,
        "tree_lambda": 0.1
      },
      "mean": 0.04825861472636461
    }
  ],
  "math_check": {
    "gram_tree_orthogonal_invariance_relative_error": 2.9398856327134126e-07
  },
  "mechanism_signature": {
    "claim": "tree aggregation preserves target set correspondence",
    "predicted": "tree-feature correlation is higher for the idea system",
    "observed_baseline_mean_tree_corr": 0.9266873237022839,
    "observed_idea_mean_tree_corr": 0.9220414828405017,
    "per_seed": [
      {
        "seed": 0,
        "baseline_tree_corr": 0.9256211589939427,
        "idea_tree_corr": 0.917472849753899,
        "baseline_mse": 0.036085259169340134,
        "idea_mse": 0.039885085076093674
      },
      {
        "seed": 1,
        "baseline_tree_corr": 0.9170005706547195,
        "idea_tree_corr": 0.924394805959731,
        "baseline_mse": 0.03398258239030838,
        "idea_mse": 0.03695981204509735
      },
      {
        "seed": 2,
        "baseline_tree_corr": 0.9286817916553544,
        "idea_tree_corr": 0.9288227539982717,
        "baseline_mse": 0.03428776562213898,
        "idea_mse": 0.038052886724472046
      },
      {
        "seed": 3,
        "baseline_tree_corr": 0.9419077791531315,
        "idea_tree_corr": 0.9076383879566745,
        "baseline_mse": 0.0314578041434288,
        "idea_mse": 0.04973628371953964
      },
      {
        "seed": 4,
        "baseline_tree_corr": 0.9288199555878167,
        "idea_tree_corr": 0.922377819116332,
        "baseline_mse": 0.034709829837083817,
        "idea_mse": 0.04566482454538345
      },
      {
        "seed": 5,
        "baseline_tree_corr": 0.9205480277200384,
        "idea_tree_corr": 0.9233275703070802,
        "baseline_mse": 0.03937392309308052,
        "idea_mse": 0.036613475531339645
      },
      {
        "seed": 6,
        "baseline_tree_corr": 0.9299053733445586,
        "idea_tree_corr": 0.9330525084728803,
        "baseline_mse": 0.035850487649440765,
        "idea_mse": 0.04028519243001938
      },
      {
        "seed": 7,
        "baseline_tree_corr": 0.9210139325087091,
        "idea_tree_corr": 0.9192451671591447,
        "baseline_mse": 0.034181930124759674,
        "idea_mse": 0.04862197861075401
      }
    ],
    "confirmed": false
  }
}