1{
  2  "bench_version": 1,
  3  "track": "dynamics",
  4  "model": "rnn_small",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "best_cfg": {
  9      "lr": 0.01
 10    },
 11    "sweep": [
 12      {
 13        "cfg": {
 14          "lr": 0.001
 15        },
 16        "mean": 0.005056460213381797
 17      },
 18      {
 19        "cfg": {
 20          "lr": 0.003
 21        },
 22        "mean": 0.0029833840089850128
 23      },
 24      {
 25        "cfg": {
 26          "lr": 0.01
 27        },
 28        "mean": 0.0014822854354861192
 29      }
 30    ],
 31    "full": {
 32      "mean": 0.0014822854354861192,
 33      "std": 0.0010530387951197904,
 34      "per_seed": [
 35        0.00411539152264595,
 36        0.0017662415048107505,
 37        0.0006022244924679399,
 38        0.0013981220545247197,
 39        0.0012625530362129211,
 40        0.0010164444101974368,
 41        0.0009094251436181366,
 42        0.000787881319411099
 43      ],
 44      "n": 8
 45    }
 46  },
 47  "idea": {
 48    "mean": 0.0021019700725446455,
 49    "std": 0.0007477001545769394,
 50    "per_seed": [
 51      0.0022576870396733284,
 52      0.0022740764543414116,
 53      0.0024945440236479044,
 54      0.0014605424366891384,
 55      0.0016900348709896207,
 56      0.0007099835784174502,
 57      0.0026052906177937984,
 58      0.003323601558804512
 59    ],
 60    "n": 8,
 61    "cfg": {
 62      "lr": 0.01,
 63      "rank": 40
 64    },
 65    "sweep": [
 66      {
 67        "cfg": {
 68          "lr": 0.01,
 69          "rank": 24
 70        },
 71        "mean": 0.003177800477715209,
 72        "std": 0.0010355345041122954,
 73        "per_seed": [
 74          0.0036525779869407415,
 75          0.004798321519047022,
 76          0.004517960362136364,
 77          0.002177618443965912,
 78          0.0021414996590465307,
 79          0.0027073805686086416,
 80          0.003489549970254302,
 81          0.0019374953117221594
 82        ],
 83        "n": 8
 84      },
 85      {
 86        "cfg": {
 87          "lr": 0.01,
 88          "rank": 32
 89        },
 90        "mean": 0.002437932969769463,
 91        "std": 0.0011340391219749532,
 92        "per_seed": [
 93          0.0013505988754332066,
 94          0.0022953480947762728,
 95          0.002628838177770376,
 96          0.002794082509353757,
 97          0.0051675527356565,
 98          0.001575573580339551,
 99          0.0017304965294897556,
100          0.0019609732553362846
101        ],
102        "n": 8
103      },
104      {
105        "cfg": {
106          "lr": 0.01,
107          "rank": 40
108        },
109        "mean": 0.0021019700725446455,
110        "std": 0.0007477001545769394,
111        "per_seed": [
112          0.0022576870396733284,
113          0.0022740764543414116,
114          0.0024945440236479044,
115          0.0014605424366891384,
116          0.0016900348709896207,
117          0.0007099835784174502,
118          0.0026052906177937984,
119          0.003323601558804512
120        ],
121        "n": 8
122      }
123    ]
124  },
125  "comparison": {
126    "delta_mean": 0.0006196846370585263,
127    "idea_wins": 2,
128    "n_pairs": 8,
129    "per_seed_diffs": [
130      -0.001857704482972622,
131      0.0005078349495306611,
132      0.0018923195311799645,
133      6.24203821644187e-05,
134      0.00042748183477669954,
135      -0.0003064608317799866,
136      0.0016958654741756618,
137      0.002535720239393413
138    ],
139    "p_value": 0.20345,
140    "mde": 0.0011741999248702343,
141    "mde_rel_pct": 79.21550713241356,
142    "verdict": "no significant win",
143    "system_worked": false,
144    "paired_delta": [
145      -0.001857704482972622,
146      0.0005078349495306611,
147      0.0018923195311799645,
148      6.24203821644187e-05,
149      0.00042748183477669954,
150      -0.0003064608317799866,
151      0.0016958654741756618,
152      0.002535720239393413
153    ]
154  },
155  "mechanism_signature": {
156    "predicted_state_cost_ratio": 0.625,
157    "observed_state_cost_ratio": 0.625,
158    "baseline_hidden": 64,
159    "idea_hidden": 40,
160    "baseline_mean_pred_observed_corr": 0.9994533416434941,
161    "idea_mean_pred_observed_corr": 0.9989892089422736,
162    "confirmed": true
163  }
164}