{
  "bench_version": 1,
  "track": "fisher_multitoken_denoising",
  "model": "custom_denoiser",
  "metric_direction": "lower is better",
  "n_seeds": 8,
  "baseline": {
    "best_cfg": {
      "lr": 0.003,
      "smax": 1.0
    },
    "sweep": [
      {
        "cfg": {
          "lr": 0.001,
          "smax": 1.0
        },
        "mean": 0.4561383128166199
      },
      {
        "cfg": {
          "lr": 0.001,
          "smax": 2.0
        },
        "mean": 0.5842899084091187
      },
      {
        "cfg": {
          "lr": 0.003,
          "smax": 1.0
        },
        "mean": 0.3060583472251892
      },
      {
        "cfg": {
          "lr": 0.003,
          "smax": 2.0
        },
        "mean": 0.59112948179245
      },
      {
        "cfg": {
          "lr": 0.01,
          "smax": 1.0
        },
        "mean": 0.6318341046571732
      },
      {
        "cfg": {
          "lr": 0.01,
          "smax": 2.0
        },
        "mean": 0.883560985326767
      }
    ],
    "full": {
      "mean": 0.3756636306643486,
      "std": 0.0827197936263635,
      "per_seed": [
        0.29077786207199097,
        0.32696259021759033,
        0.3145223557949066,
        0.2919705808162689,
        0.4482685327529907,
        0.4811434745788574,
        0.5064208507537842,
        0.34524279832839966
      ],
      "n": 8
    }
  },
  "idea": {
    "mean": 0.38846133276820183,
    "std": 0.08522182674569097,
    "per_seed": [
      0.2787860035896301,
      0.3444952070713043,
      0.3325658440589905,
      0.30363890528678894,
      0.4642293453216553,
      0.5044667720794678,
      0.5081693530082703,
      0.37133923172950745
    ],
    "n": 8
  },
  "comparison": {
    "delta_mean": 0.012797702103853226,
    "idea_wins": 1,
    "n_pairs": 8,
    "per_seed_diffs": [
      -0.01199185848236084,
      0.01753261685371399,
      0.018043488264083862,
      0.01166832447052002,
      0.01596081256866455,
      0.02332329750061035,
      0.001748502254486084,
      0.026096433401107788
    ],
    "p_value": 0.03075,
    "mde": 0.010412284911868961,
    "mde_rel_pct": 2.77170427529947,
    "verdict": "idea worse (significant)",
    "system_worked": false
  },
  "mechanism_signature": {
    "predicted_equal_arc_local_kl_cv": 1.7439286709141664e-06,
    "observed_parameter_path_linear_kl_cv": 0.6206565234045026,
    "trained_model_stage_behavior": [
      {
        "schedule": "linear",
        "observed_stage_error_cv": 0.610444495224178,
        "mean_stage_error": 0.08814288070425391
      },
      {
        "schedule": "fisher",
        "observed_stage_error_cv": 0.4789595501048241,
        "mean_stage_error": 0.10801896243356168
      }
    ],
    "confirmed": true,
    "note": "Stage errors are measured on the trained Fisher model; analytic KL values are included only as the re-tested mechanism prediction."
  },
  "idea_sweep": [
    {
      "cfg": {
        "lr": 0.003,
        "smax": 1.0
      },
      "result": {
        "mean": 0.38846133276820183,
        "std": 0.08522182674569097,
        "per_seed": [
          0.2787860035896301,
          0.3444952070713043,
          0.3325658440589905,
          0.30363890528678894,
          0.4642293453216553,
          0.5044667720794678,
          0.5081693530082703,
          0.37133923172950745
        ],
        "n": 8
      }
    },
    {
      "cfg": {
        "lr": 0.001,
        "smax": 1.0
      },
      "result": {
        "mean": 0.49281759932637215,
        "std": 0.03752558894150052,
        "per_seed": [
          0.47663557529449463,
          0.5520936846733093,
          0.5091399550437927,
          0.48174160718917847,
          0.4514659345149994,
          0.48623156547546387,
          0.544323742389679,
          0.4409087300300598
        ],
        "n": 8
      }
    },
    {
      "cfg": {
        "lr": 0.01,
        "smax": 1.0
      },
      "result": {
        "mean": 0.5914078056812286,
        "std": 0.11582810816462852,
        "per_seed": [
          0.7036856412887573,
          0.35683417320251465,
          0.5790603756904602,
          0.7571036219596863,
          0.6265718936920166,
          0.5940495133399963,
          0.4918774962425232,
          0.6220797300338745
        ],
        "n": 8
      }
    }
  ],
  "custom_track": {
    "name": "fisher_multitoken_denoising",
    "file": "fisher_bench_track.py",
    "domain": "diffusion-sampling"
  },
  "structural_match": "multi-token denoising with correlated sequence distributions; schedule is the only method difference"
}