{ "bench_version": 1, "track": "dynamics", "model": "rnn_small", "metric_direction": "lower is better", "n_seeds": 8, "baseline": { "best_cfg": { "lr": 0.009 }, "sweep": [ { "cfg": { "lr": 0.001 }, "mean": 3.668733984341088e-05 }, { "cfg": { "lr": 0.003 }, "mean": 8.922007737055537e-06 }, { "cfg": { "lr": 0.009 }, "mean": 4.542415013020218e-06 } ], "full": { "mean": 5.152216147052968e-06, "std": 1.9023458106411929e-06, "per_seed": [ 4.7863295549177565e-06, 5.878216143173631e-06, 3.588439312807168e-06, 3.7736158446932677e-06, 6.708174169034464e-06, 8.705701475264505e-06, 5.577248430199688e-06, 2.2000042463332647e-06 ], "n": 8 } }, "idea": { "mean": 4.707826093408585e-06, "std": 2.320462925135068e-06, "per_seed": [ 2.252499598398572e-06, 3.3756230095605133e-06, 3.7817651445948286e-06, 8.986970897240099e-06, 3.874472895404324e-06, 3.393555971342721e-06, 8.273613275378011e-06, 3.7241079553496093e-06 ], "n": 8 }, "comparison": { "delta_mean": -4.4439005364438344e-07, "idea_wins": 4, "n_pairs": 8, "per_seed_diffs": [ -2.5338299565191846e-06, -2.502593133613118e-06, 1.9332583178766072e-07, 5.213355052546831e-06, -2.8337012736301403e-06, -5.3121455039217835e-06, 2.696364845178323e-06, 1.5241037090163445e-06 ], "p_value": 0.7344, "mde": 2.8993112919196206e-06, "mde_rel_pct": 56.27309121295322, "verdict": "no significant win", "system_worked": false }, "idea_sweep": [ { "cfg": { "lr": 0.001 }, "result": { "mean": 3.7107075286257896e-05, "std": 6.4375009711858975e-06, "per_seed": [ 2.8180171284475364e-05, 4.052478107041679e-05, 3.2498250220669433e-05, 5.090365812066011e-05, 3.462454697000794e-05, 3.355262742843479e-05, 3.641599687398411e-05, 4.015657032141462e-05 ], "n": 8 } }, { "cfg": { "lr": 0.003 }, "result": { "mean": 1.0382680102338782e-05, "std": 3.1424418893375147e-06, "per_seed": [ 1.1441469723649789e-05, 9.949631021299865e-06, 1.4452817595156375e-05, 1.5155372238950804e-05, 8.438014447165187e-06, 5.9799863265652675e-06, 1.1092530257883482e-05, 6.551619208039483e-06 ], "n": 8 } }, { "cfg": { "lr": 0.009 }, "result": { "mean": 4.707826093408585e-06, "std": 2.320462925135068e-06, "per_seed": [ 2.252499598398572e-06, 3.3756230095605133e-06, 3.7817651445948286e-06, 8.986970897240099e-06, 3.874472895404324e-06, 3.393555971342721e-06, 8.273613275378011e-06, 3.7241079553496093e-06 ], "n": 8 } } ], "track_justification": "Dynamics is the built-in control/stability track and contains delayed actuated pendulum rollouts.", "mechanism_signature": { "delay": 4, "trained_models": true, "measured_on": "official dynamics test windows", "mean_predicted_terminal_state_shift": 0.21677356958389282, "predicted_signature": "finite-horizon transport prediction changes stale terminal state", "confirmed": true } }