Label-Free Bayesian Truth Serum Reward / bts_track.py

Mechanism confirmed, baseline not beaten

Raw ⬇ ZIP
 1import numpy as np
 2
 3META = {
 4    "name": "belief_sensitive_peer_prediction",
 5    "domain": "rlhf",
 6    "description": "Binary latent-fact classification with noisy evidence and misleading user-pressure features; supports peer-prediction group rewards.",
 7}
 8
 9
10def get_dataset(seed, n_train, n_test):
11    rng = np.random.RandomState(seed)
12
13    def make(n):
14        truth = rng.randint(0, 2, size=n)
15        evidence = (2 * truth - 1) + rng.normal(0, 0.75, size=n)
16        pressure = (1 - 2 * truth) + rng.normal(0, 0.25, size=n)
17        pressure *= (rng.rand(n) > 0.18)
18        x = np.stack([evidence, pressure], axis=1).astype(np.float32)
19        return x, truth.astype(np.int64)
20
21    xtr, ytr = make(n_train)
22    xte, yte = make(n_test)
23    return {
24        "xtr": xtr, "ytr": ytr, "xte": xte, "yte": yte,
25        "task": "classification", "metric": "err", "out_dim": 2,
26    }