{
  "bench_version": 1,
  "track": "categorical_persistent_sequences",
  "model": "mlp_token_logits",
  "metric_direction": "lower is better",
  "n_seeds": 8,
  "baseline": {
    "best_cfg": {
      "lr": 0.003,
      "temp": 0.8
    },
    "sweep": [
      {
        "cfg": {
          "lr": 0.001,
          "temp": 0.8
        },
        "mean": 0.49406249999999996
      },
      {
        "cfg": {
          "lr": 0.001,
          "temp": 1.0
        },
        "mean": 0.49406249999999996
      },
      {
        "cfg": {
          "lr": 0.001,
          "temp": 1.2
        },
        "mean": 0.49406249999999996
      },
      {
        "cfg": {
          "lr": 0.003,
          "temp": 0.8
        },
        "mean": 0.490625
      },
      {
        "cfg": {
          "lr": 0.003,
          "temp": 1.0
        },
        "mean": 0.490625
      },
      {
        "cfg": {
          "lr": 0.003,
          "temp": 1.2
        },
        "mean": 0.490625
      },
      {
        "cfg": {
          "lr": 0.01,
          "temp": 0.8
        },
        "mean": 0.49687499999999996
      },
      {
        "cfg": {
          "lr": 0.01,
          "temp": 1.0
        },
        "mean": 0.49687499999999996
      },
      {
        "cfg": {
          "lr": 0.01,
          "temp": 1.2
        },
        "mean": 0.49687499999999996
      }
    ],
    "full": {
      "mean": 0.49445312500000005,
      "std": 0.015781830125475786,
      "per_seed": [
        0.460625,
        0.498125,
        0.514375,
        0.489375,
        0.483125,
        0.50875,
        0.501875,
        0.499375
      ],
      "n": 8
    }
  },
  "idea": {
    "mean": 0.48640625000000004,
    "std": 0.013064406260427612,
    "per_seed": [
      0.480625,
      0.503125,
      0.48625,
      0.49875,
      0.47375,
      0.464375,
      0.4825,
      0.501875
    ],
    "n": 8
  },
  "comparison": {
    "delta_mean": -0.008046875000000002,
    "idea_wins": 4,
    "n_pairs": 8,
    "per_seed_diffs": [
      0.020000000000000018,
      0.00500000000000006,
      -0.02812500000000001,
      0.009375000000000022,
      -0.009375000000000022,
      -0.04437500000000005,
      -0.019374999999999976,
      0.0024999999999999467
    ],
    "p_value": 0.3411,
    "mde": 0.01794619570947467,
    "mde_rel_pct": 3.629503951355282,
    "verdict": "no significant win",
    "system_worked": false
  },
  "mechanism_signature": {
    "custom_track": {
      "name": "categorical_persistent_sequences",
      "file": "categorical_track.py",
      "domain": "diffusion-sampling"
    },
    "idea_config": {
      "lr": 0.01,
      "temp": 1.0,
      "rho": 0.0
    },
    "idea_sweep": [
      {
        "cfg": {
          "lr": 0.001,
          "temp": 0.8,
          "rho": 0.0
        },
        "result": {
          "mean": 0.4915625,
          "std": 0.007335434121440939,
          "per_seed": [
            0.49375,
            0.495,
            0.488125,
            0.475,
            0.49,
            0.495625,
            0.501875,
            0.493125
          ],
          "n": 8
        }
      },
      {
        "cfg": {
          "lr": 0.001,
          "temp": 0.8,
          "rho": 0.5
        },
        "result": {
          "mean": 0.49539062500000003,
          "std": 0.012062408921288268,
          "per_seed": [
            0.49375,
            0.523125,
            0.493125,
            0.49375,
            0.4775,
            0.49875,
            0.495,
            0.488125
          ],
          "n": 8
        }
      },
      {
        "cfg": {
          "lr": 0.001,
          "temp": 0.8,
          "rho": 0.75
        },
        "result": {
          "mean": 0.495390625,
          "std": 0.011030574961640704,
          "per_seed": [
            0.49125,
            0.515625,
            0.493125,
            0.498125,
            0.47875,
            0.5075,
            0.485,
            0.49375
          ],
          "n": 8
        }
      },
      {
        "cfg": {
          "lr": 0.001,
          "temp": 1.0,
          "rho": 0.0
        },
        "result": {
          "mean": 0.49109375,
          "std": 0.007842131307080995,
          "per_seed": [
            0.493125,
            0.495,
            0.48625,
            0.473125,
            0.49125,
            0.496875,
            0.500625,
            0.4925
          ],
          "n": 8
        }
      },
      {
        "cfg": {
          "lr": 0.001,
          "temp": 1.0,
          "rho": 0.5
        },
        "result": {
          "mean": 0.495859375,
          "std": 0.012342513783033639,
          "per_seed": [
            0.4925,
            0.52375,
            0.49375,
            0.495,
            0.47625,
            0.499375,
            0.495625,
            0.490625
          ],
          "n": 8
        }
      },
      {
        "cfg": {
          "lr": 0.001,
          "temp": 1.0,
          "rho": 0.75
        },
        "result": {
          "mean": 0.49484375,
          "std": 0.011214135541248827,
          "per_seed": [
            0.491875,
            0.515625,
            0.493125,
            0.49125,
            0.47875,
            0.50875,
            0.485,
            0.494375
          ],
          "n": 8
        }
      },
      {
        "cfg": {
          "lr": 0.001,
          "temp": 1.2,
          "rho": 0.0
        },
        "result": {
          "mean": 0.49007812500000003,
          "std": 0.007351641371787327,
          "per_seed": [
            0.49125,
            0.49625,
            0.48625,
            0.474375,
            0.490625,
            0.49375,
            0.500625,
            0.4875
          ],
          "n": 8
        }
      },
      {
        "cfg": {
          "lr": 0.001,
          "temp": 1.2,
          "rho": 0.5
        },
        "result": {
          "mean": 0.49554687499999994,
          "std": 0.011727861041527343,
          "per_seed": [
            0.491875,
            0.5225,
            0.49375,
            0.493125,
            0.4775,
            0.498125,
            0.49625,
            0.49125
          ],
          "n": 8
        }
      },
      {
        "cfg": {
          "lr": 0.001,
          "temp": 1.2,
          "rho": 0.75
        },
        "result": {
          "mean": 0.49484375,
          "std": 0.01113548487662301,
          "per_seed": [
            0.49125,
            0.515625,
            0.495625,
            0.491875,
            0.478125,
            0.5075,
            0.485,
            0.49375
          ],
          "n": 8
        }
      },
      {
        "cfg": {
          "lr": 0.003,
          "temp": 0.8,
          "rho": 0.0
        },
        "result": {
          "mean": 0.48953125,
          "std": 0.005587985856952403,
          "per_seed": [
            0.491875,
            0.4975,
            0.49125,
            0.486875,
            0.4775,
            0.490625,
            0.486875,
            0.49375
          ],
          "n": 8
        }
      },
      {
        "cfg": {
          "lr": 0.003,
          "temp": 0.8,
          "rho": 0.5
        },
        "result": {
          "mean": 0.49203125000000003,
          "std": 0.013609904194648118,
          "per_seed": [
            0.503125,
            0.51375,
            0.48375,
            0.496875,
            0.464375,
            0.495625,
            0.490625,
            0.488125
          ],
          "n": 8
        }
      },
      {
        "cfg": {
          "lr": 0.003,
          "temp": 0.8,
          "rho": 0.75
        },
        "result": {
          "mean": 0.494921875,
          "std": 0.00963411449534284,
          "per_seed": [
            0.488125,
            0.514375,
            0.49,
            0.49375,
            0.483125,
            0.505625,
            0.48875,
            0.495625
          ],
          "n": 8
        }
      },
      {
        "cfg": {
          "lr": 0.003,
          "temp": 1.0,
          "rho": 0.0
        },
        "result": {
          "mean": 0.488984375,
          "std": 0.0071393598441579515,
          "per_seed": [
            0.494375,
            0.496875,
            0.49,
            0.484375,
            0.475625,
            0.486875,
            0.485,
            0.49875
          ],
          "n": 8
        }
      },
      {
        "cfg": {
          "lr": 0.003,
          "temp": 1.0,
          "rho": 0.5
        },
        "result": {
          "mean": 0.491328125,
          "std": 0.013193413588202828,
          "per_seed": [
            0.5025,
            0.513125,
            0.481875,
            0.494375,
            0.465,
            0.491875,
            0.49125,
            0.490625
          ],
          "n": 8
        }
      },
      {
        "cfg": {
          "lr": 0.003,
          "temp": 1.0,
          "rho": 0.75
        },
        "result": {
          "mean": 0.49421875,
          "std": 0.009456675469608747,
          "per_seed": [
            0.489375,
            0.510625,
            0.485,
            0.495,
            0.483125,
            0.5075,
            0.488125,
            0.495
          ],
          "n": 8
        }
      },
      {
        "cfg": {
          "lr": 0.003,
          "temp": 1.2,
          "rho": 0.0
        },
        "result": {
          "mean": 0.490234375,
          "std": 0.0072074282156935204,
          "per_seed": [
            0.495625,
            0.5,
            0.490625,
            0.486875,
            0.475,
            0.490625,
            0.486875,
            0.49625
          ],
          "n": 8
        }
      },
      {
        "cfg": {
          "lr": 0.003,
          "temp": 1.2,
          "rho": 0.5
        },
        "result": {
          "mean": 0.49203125000000003,
          "std": 0.013049447725382855,
          "per_seed": [
            0.50125,
            0.5125,
            0.48375,
            0.495625,
            0.464375,
            0.493125,
            0.495,
            0.490625
          ],
          "n": 8
        }
      },
      {
        "cfg": {
          "lr": 0.003,
          "temp": 1.2,
          "rho": 0.75
        },
        "result": {
          "mean": 0.494609375,
          "std": 0.010448616366982527,
          "per_seed": [
            0.494375,
            0.51125,
            0.4825,
            0.495625,
            0.48,
            0.50875,
            0.48875,
            0.495625
          ],
          "n": 8
        }
      },
      {
        "cfg": {
          "lr": 0.01,
          "temp": 0.8,
          "rho": 0.0
        },
        "result": {
          "mean": 0.487890625,
          "std": 0.01414882359012138,
          "per_seed": [
            0.48,
            0.51,
            0.488125,
            0.503125,
            0.4725,
            0.468125,
            0.48125,
            0.5
          ],
          "n": 8
        }
      },
      {
        "cfg": {
          "lr": 0.01,
          "temp": 0.8,
          "rho": 0.5
        },
        "result": {
          "mean": 0.494140625,
          "std": 0.014337374331947086,
          "per_seed": [
            0.49625,
            0.525625,
            0.485,
            0.495,
            0.47625,
            0.48375,
            0.4875,
            0.50375
          ],
          "n": 8
        }
      },
      {
        "cfg": {
          "lr": 0.01,
          "temp": 0.8,
          "rho": 0.75
        },
        "result": {
          "mean": 0.494375,
          "std": 0.012480453467322404,
          "per_seed": [
            0.48,
            0.5225,
            0.485,
            0.49875,
            0.490625,
            0.490625,
            0.48625,
            0.50125
          ],
          "n": 8
        }
      },
      {
        "cfg": {
          "lr": 0.01,
          "temp": 1.0,
          "rho": 0.0
        },
        "result": {
          "mean": 0.48640625000000004,
          "std": 0.013064406260427612,
          "per_seed": [
            0.480625,
            0.503125,
            0.48625,
            0.49875,
            0.47375,
            0.464375,
            0.4825,
            0.501875
          ],
          "n": 8
        }
      },
      {
        "cfg": {
          "lr": 0.01,
          "temp": 1.0,
          "rho": 0.5
        },
        "result": {
          "mean": 0.4946875,
          "std": 0.012751378601939463,
          "per_seed": [
            0.495625,
            0.518125,
            0.48625,
            0.498125,
            0.475625,
            0.48625,
            0.48875,
            0.50875
          ],
          "n": 8
        }
      },
      {
        "cfg": {
          "lr": 0.01,
          "temp": 1.0,
          "rho": 0.75
        },
        "result": {
          "mean": 0.495,
          "std": 0.010798220339481851,
          "per_seed": [
            0.481875,
            0.518125,
            0.48375,
            0.4975,
            0.49375,
            0.493125,
            0.489375,
            0.5025
          ],
          "n": 8
        }
      },
      {
        "cfg": {
          "lr": 0.01,
          "temp": 1.2,
          "rho": 0.0
        },
        "result": {
          "mean": 0.48726562500000004,
          "std": 0.012251185768503196,
          "per_seed": [
            0.483125,
            0.499375,
            0.489375,
            0.495625,
            0.47375,
            0.464375,
            0.489375,
            0.503125
          ],
          "n": 8
        }
      },
      {
        "cfg": {
          "lr": 0.01,
          "temp": 1.2,
          "rho": 0.5
        },
        "result": {
          "mean": 0.49382812499999995,
          "std": 0.013329645282578786,
          "per_seed": [
            0.49625,
            0.516875,
            0.485,
            0.49375,
            0.47375,
            0.485625,
            0.488125,
            0.51125
          ],
          "n": 8
        }
      },
      {
        "cfg": {
          "lr": 0.01,
          "temp": 1.2,
          "rho": 0.75
        },
        "result": {
          "mean": 0.49671875,
          "std": 0.010325512623473003,
          "per_seed": [
            0.4825,
            0.51875,
            0.486875,
            0.495,
            0.49625,
            0.494375,
            0.495625,
            0.504375
          ],
          "n": 8
        }
      }
    ],
    "mechanism_signature": {
      "prediction": "Gaussian refresh sign correlation is 2 asin(rho)/pi",
      "rho": 0.75,
      "rows": [
        {
          "predicted_sign_corr": 0.5398930876747683,
          "observed_sign_corr": 0.060386206896551727
        },
        {
          "predicted_sign_corr": 0.5398930876747683,
          "observed_sign_corr": 0.06482068965517242
        },
        {
          "predicted_sign_corr": 0.5398930876747683,
          "observed_sign_corr": 0.07049655172413793
        },
        {
          "predicted_sign_corr": 0.5398930876747683,
          "observed_sign_corr": 0.06498448275862069
        }
      ],
      "mean_abs_error": 0.4747211049161476,
      "tolerance": 0.04,
      "confirmed": false
    }
  }
}