Inverse-Square Fractional Attention / repeat.py

Failed on benchmark

Raw ⬇ ZIP
 1import json, random, numpy as np, torch
 2from experiment import AttnRegressor, make_data
 3
 4def one(seed, device):
 5    random.seed(seed); np.random.seed(seed); torch.manual_seed(seed)
 6    tr_x,tr_y=make_data(256); te_x,te_y=make_data(64)
 7    tr_x,tr_y,te_x,te_y=[z.to(device) for z in (tr_x,tr_y,te_x,te_y)]
 8    out={}
 9    for biased in (False,True):
10        torch.manual_seed(seed); m=AttnRegressor(biased).to(device)
11        opt=torch.optim.Adam(m.parameters(),lr=3e-3); mse=torch.nn.MSELoss()
12        for _ in range(180):
13            ix=torch.randint(0,256,(16,),device=device)
14            loss=mse(m(tr_x[ix]),tr_y[ix]); opt.zero_grad(); loss.backward(); opt.step()
15        with torch.no_grad():
16            pred=m(te_x); err=(pred-te_y).square().squeeze(-1)
17            r=(te_x.square().sum(-1)).sqrt()
18            near=err[r<.35].mean().item(); far=err[r>.7].mean().item()
19            out['idea' if biased else 'baseline']=[mse(pred,te_y).item(),near,far]
20    return out
21
22def main():
23    device='cuda' if torch.cuda.is_available() else 'cpu'
24    try:
25        vals=[one(s,device) for s in (17,23,41)]
26    except Exception:
27        device='cpu'; vals=[one(s,device) for s in (17,23,41)]
28    print(json.dumps({'device':device,'seeds':[17,23,41],'columns':['overall_mse','near_origin_mse','far_mse'],'runs':vals},indent=2))
29if __name__=='__main__': main()