{
  "bench_version": 1,
  "track": "belief_sensitive_peer_prediction",
  "model": "mlp_tiny",
  "metric_direction": "lower is better",
  "n_seeds": 8,
  "baseline": {
    "best_cfg": {
      "lr": 0.003
    },
    "sweep": [
      {
        "cfg": {
          "lr": 0.001
        },
        "mean": 0.016249999636784196
      },
      {
        "cfg": {
          "lr": 0.003
        },
        "mean": 0.01499999943189323
      },
      {
        "cfg": {
          "lr": 0.01
        },
        "mean": 0.019999999552965164
      }
    ],
    "full": {
      "mean": 0.016874999273568392,
      "std": 0.007473912417713013,
      "per_seed": [
        0.009999999776482582,
        0.019999999552965164,
        0.02499999850988388,
        0.004999999888241291,
        0.014999999664723873,
        0.02499999850988388,
        0.009999999776482582,
        0.02499999850988388
      ],
      "n": 8
    }
  },
  "idea": {
    "mean": 0.3287499975413084,
    "std": 0.18487749114388957,
    "per_seed": [
      0.10999999940395355,
      0.16500000655651093,
      0.6800000071525574,
      0.25999999046325684,
      0.17000000178813934,
      0.3050000071525574,
      0.5199999809265137,
      0.41999998688697815
    ],
    "n": 8
  },
  "comparison": {
    "delta_mean": 0.31187499826774,
    "idea_wins": 0,
    "n_pairs": 8,
    "per_seed_diffs": [
      0.09999999962747097,
      0.14500000700354576,
      0.6550000086426735,
      0.25499999057501554,
      0.15500000212341547,
      0.2800000086426735,
      0.5099999811500311,
      0.39499998837709427
    ],
    "p_value": 0.0081,
    "mde": 0.16284215222207377,
    "mde_rel_pct": 964.9905732270832,
    "verdict": "idea worse (significant)",
    "system_worked": false
  },
  "mechanism_signature": {
    "custom_track": {
      "name": "belief_sensitive_peer_prediction",
      "file": "bts_track.py",
      "domain": "rlhf"
    },
    "idea_sweep": [
      {
        "cfg": {
          "lr": 0.001
        },
        "result": {
          "mean": 0.4612500015646219,
          "std": 0.25068343891180656,
          "per_seed": [
            0.375,
            0.675000011920929,
            0.8999999761581421,
            0.5400000214576721,
            0.4050000011920929,
            0.17499999701976776,
            0.5550000071525574,
            0.06499999761581421
          ],
          "n": 8
        }
      },
      {
        "cfg": {
          "lr": 0.003
        },
        "result": {
          "mean": 0.3287499975413084,
          "std": 0.18487749114388957,
          "per_seed": [
            0.10999999940395355,
            0.16500000655651093,
            0.6800000071525574,
            0.25999999046325684,
            0.17000000178813934,
            0.3050000071525574,
            0.5199999809265137,
            0.41999998688697815
          ],
          "n": 8
        }
      },
      {
        "cfg": {
          "lr": 0.01
        },
        "result": {
          "mean": 0.5187499970197678,
          "std": 0.04385701432651679,
          "per_seed": [
            0.5149999856948853,
            0.45500001311302185,
            0.5099999904632568,
            0.5299999713897705,
            0.6150000095367432,
            0.47999998927116394,
            0.5099999904632568,
            0.5350000262260437
          ],
          "n": 8
        }
      }
    ],
    "mechanism_signature": {
      "group_size": 8,
      "n_model_completions": 1024,
      "predicted_mean_frequency": 0.5131519436836243,
      "observed_mean_frequency": 0.5771484375,
      "predicted_observed_correlation": 0.400832738120897,
      "log_ratio_max_error": 0.0,
      "confirmed": true
    }
  }
}