Strongly-convex superwind attention / results.json

Mechanism failed

Raw ⬇ ZIP
 1{
 2  "device": "cuda",
 3  "math_check": {
 4    "linear_profile_g_min": 0.9999999403953552,
 5    "linear_profile_g_max": 1.0,
 6    "linear_profile_hessian_min": 0.6939469575881958,
 7    "bad_profile_hessian_min": -11.702886581420898
 8  },
 9  "baseline_dot": {
10    "final_loss": 0.01768084429204464,
11    "mean_last20": 0.019677276071161032,
12    "loss_std_last50": 0.004114333493313028,
13    "max_grad": 0.9343541860580444,
14    "final_negative_g_fraction": 0.0,
15    "mean_attention_entropy": 0.7476468443870544
16  },
17  "unconstrained_metric": {
18    "final_loss": 0.01860710233449936,
19    "mean_last20": 0.02084847129881382,
20    "loss_std_last50": 0.004466183089062249,
21    "max_grad": 0.9493426084518433,
22    "final_negative_g_fraction": 0.0,
23    "mean_attention_entropy": 8.152075793077529e-06
24  },
25  "constrained_metric": {
26    "final_loss": 0.018606653437018394,
27    "mean_last20": 0.020852698013186455,
28    "loss_std_last50": 0.004467031694072602,
29    "max_grad": 0.9493432641029358,
30    "final_negative_g_fraction": 0.0,
31    "mean_attention_entropy": 2.02627506951103e-06
32  }
33}