Sparse Lyapunov Search for Safe Optimizer Hyperparameters / evaluate.py
Mechanism failed
1import json
2import numpy as np
3from sparse_lyapunov import rho_scalar, run_experiment, verify_math
4
5
6def summarize_trials(seeds=range(10)):
7 rows = [run_experiment(seed=int(s), steps=180) for s in seeds]
8 b20 = np.array([r['baseline']['loss_20pct'] for r in rows])
9 i20 = np.array([r['idea']['loss_20pct'] for r in rows])
10 bf = np.array([r['baseline']['final_loss'] for r in rows])
11 inf = np.array([r['idea']['final_loss'] for r in rows])
12 bs = np.array([r['baseline']['spikes'] for r in rows])
13 ins = np.array([r['idea']['spikes'] for r in rows])
14 checked = []
15 for r in rows:
16 for cert, curv in zip(r['idea']['certificates'], r['curvatures']):
17 checked.append(max(rho_scalar(cert['eta'], cert['beta'], float(v)) for v in curv))
18 return {
19 'math': verify_math(), 'n_trials': len(rows),
20 'baseline_mean_loss_20pct': float(b20.mean()),
21 'idea_mean_loss_20pct': float(i20.mean()),
22 'baseline_mean_final_loss': float(bf.mean()),
23 'idea_mean_final_loss': float(inf.mean()),
24 'baseline_mean_spikes': float(bs.mean()),
25 'idea_mean_spikes': float(ins.mean()),
26 'early_loss_ratio_idea_over_baseline': float(np.mean(i20 / b20)),
27 'all_reported_endpoint_rhos_lt_1': bool(np.all(np.array(checked) < 1.0)),
28 'max_checked_endpoint_rho': float(max(checked)),
29 'per_seed': [{'seed': int(s), 'baseline_loss_20pct': float(a),
30 'idea_loss_20pct': float(b), 'baseline_spikes': int(c),
31 'idea_spikes': int(d)} for s,a,b,c,d in zip(seeds,b20,i20,bs,ins)]
32 }
33
34if __name__ == '__main__':
35 print(json.dumps(summarize_trials(), indent=2))