1{
  2  "bench_version": 1,
  3  "track": "belief_sensitive_peer_prediction",
  4  "model": "mlp_tiny",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "best_cfg": {
  9      "lr": 0.003
 10    },
 11    "sweep": [
 12      {
 13        "cfg": {
 14          "lr": 0.001
 15        },
 16        "mean": 0.016249999636784196
 17      },
 18      {
 19        "cfg": {
 20          "lr": 0.003
 21        },
 22        "mean": 0.01499999943189323
 23      },
 24      {
 25        "cfg": {
 26          "lr": 0.01
 27        },
 28        "mean": 0.019999999552965164
 29      }
 30    ],
 31    "full": {
 32      "mean": 0.016874999273568392,
 33      "std": 0.007473912417713013,
 34      "per_seed": [
 35        0.009999999776482582,
 36        0.019999999552965164,
 37        0.02499999850988388,
 38        0.004999999888241291,
 39        0.014999999664723873,
 40        0.02499999850988388,
 41        0.009999999776482582,
 42        0.02499999850988388
 43      ],
 44      "n": 8
 45    }
 46  },
 47  "idea": {
 48    "mean": 0.3287499975413084,
 49    "std": 0.18487749114388957,
 50    "per_seed": [
 51      0.10999999940395355,
 52      0.16500000655651093,
 53      0.6800000071525574,
 54      0.25999999046325684,
 55      0.17000000178813934,
 56      0.3050000071525574,
 57      0.5199999809265137,
 58      0.41999998688697815
 59    ],
 60    "n": 8
 61  },
 62  "comparison": {
 63    "delta_mean": 0.31187499826774,
 64    "idea_wins": 0,
 65    "n_pairs": 8,
 66    "per_seed_diffs": [
 67      0.09999999962747097,
 68      0.14500000700354576,
 69      0.6550000086426735,
 70      0.25499999057501554,
 71      0.15500000212341547,
 72      0.2800000086426735,
 73      0.5099999811500311,
 74      0.39499998837709427
 75    ],
 76    "p_value": 0.0081,
 77    "mde": 0.16284215222207377,
 78    "mde_rel_pct": 964.9905732270832,
 79    "verdict": "idea worse (significant)",
 80    "system_worked": false
 81  },
 82  "mechanism_signature": {
 83    "custom_track": {
 84      "name": "belief_sensitive_peer_prediction",
 85      "file": "bts_track.py",
 86      "domain": "rlhf"
 87    },
 88    "idea_sweep": [
 89      {
 90        "cfg": {
 91          "lr": 0.001
 92        },
 93        "result": {
 94          "mean": 0.4612500015646219,
 95          "std": 0.25068343891180656,
 96          "per_seed": [
 97            0.375,
 98            0.675000011920929,
 99            0.8999999761581421,
100            0.5400000214576721,
101            0.4050000011920929,
102            0.17499999701976776,
103            0.5550000071525574,
104            0.06499999761581421
105          ],
106          "n": 8
107        }
108      },
109      {
110        "cfg": {
111          "lr": 0.003
112        },
113        "result": {
114          "mean": 0.3287499975413084,
115          "std": 0.18487749114388957,
116          "per_seed": [
117            0.10999999940395355,
118            0.16500000655651093,
119            0.6800000071525574,
120            0.25999999046325684,
121            0.17000000178813934,
122            0.3050000071525574,
123            0.5199999809265137,
124            0.41999998688697815
125          ],
126          "n": 8
127        }
128      },
129      {
130        "cfg": {
131          "lr": 0.01
132        },
133        "result": {
134          "mean": 0.5187499970197678,
135          "std": 0.04385701432651679,
136          "per_seed": [
137            0.5149999856948853,
138            0.45500001311302185,
139            0.5099999904632568,
140            0.5299999713897705,
141            0.6150000095367432,
142            0.47999998927116394,
143            0.5099999904632568,
144            0.5350000262260437
145          ],
146          "n": 8
147        }
148      }
149    ],
150    "mechanism_signature": {
151      "group_size": 8,
152      "n_model_completions": 1024,
153      "predicted_mean_frequency": 0.5131519436836243,
154      "observed_mean_frequency": 0.5771484375,
155      "predicted_observed_correlation": 0.400832738120897,
156      "log_ratio_max_error": 0.0,
157      "confirmed": true
158    }
159  }
160}