1{
  2  "bench_version": 1,
  3  "track": "dynamics",
  4  "model": "rnn_small",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "best_cfg": {
  9      "lr": 0.006,
 10      "weight_decay": 0.0001
 11    },
 12    "sweep": [
 13      {
 14        "cfg": {
 15          "lr": 0.0015,
 16          "weight_decay": 0.0
 17        },
 18        "mean": 0.005481100233737379
 19      },
 20      {
 21        "cfg": {
 22          "lr": 0.0015,
 23          "weight_decay": 0.0001
 24        },
 25        "mean": 0.005474042263813317
 26      },
 27      {
 28        "cfg": {
 29          "lr": 0.003,
 30          "weight_decay": 0.0
 31        },
 32        "mean": 0.003975238680141047
 33      },
 34      {
 35        "cfg": {
 36          "lr": 0.003,
 37          "weight_decay": 0.0001
 38        },
 39        "mean": 0.00394076865632087
 40      },
 41      {
 42        "cfg": {
 43          "lr": 0.006,
 44          "weight_decay": 0.0
 45        },
 46        "mean": 0.002146739774616435
 47      },
 48      {
 49        "cfg": {
 50          "lr": 0.006,
 51          "weight_decay": 0.0001
 52        },
 53        "mean": 0.002088654466206208
 54      }
 55    ],
 56    "full": {
 57      "mean": 0.0022117759071988985,
 58      "std": 0.0004126881098400608,
 59      "per_seed": [
 60        0.002800536109134555,
 61        0.001347046927548945,
 62        0.001825663261115551,
 63        0.0023813715670257807,
 64        0.0023169058840721846,
 65        0.0024658460170030594,
 66        0.0022247303277254105,
 67        0.002332107163965702
 68      ],
 69      "n": 8
 70    }
 71  },
 72  "idea": {
 73    "mean": 0.0020463809778448194,
 74    "std": 0.0006329416993684799,
 75    "per_seed": [
 76      0.003160907654091716,
 77      0.0019281062996014953,
 78      0.0024306366685777903,
 79      0.0027447089087218046,
 80      0.0015351816546171904,
 81      0.0011941296979784966,
 82      0.0018756426870822906,
 83      0.001501734252087772
 84    ],
 85    "n": 8,
 86    "best_cfg": {
 87      "lr": 0.006,
 88      "weight_decay": 0.0001,
 89      "cert_weight": 0.01
 90    },
 91    "sweep": [
 92      {
 93        "cfg": {
 94          "lr": 0.006,
 95          "weight_decay": 0.0001,
 96          "cert_weight": 0.01
 97        },
 98        "mean": 0.0025660898827482015
 99      },
100      {
101        "cfg": {
102          "lr": 0.006,
103          "weight_decay": 0.0001,
104          "cert_weight": 0.05
105        },
106        "mean": 0.0025660898827482015
107      },
108      {
109        "cfg": {
110          "lr": 0.006,
111          "weight_decay": 0.0001,
112          "cert_weight": 0.2
113        },
114        "mean": 0.0025660898827482015
115      }
116    ]
117  },
118  "comparison": {
119    "delta_mean": -0.00016539492935407907,
120    "idea_wins": 4,
121    "n_pairs": 8,
122    "per_seed_diffs": [
123      0.00036037154495716095,
124      0.0005810593720525503,
125      0.0006049734074622393,
126      0.00036333734169602394,
127      -0.0007817242294549942,
128      -0.0012717163190245628,
129      -0.0003490876406431198,
130      -0.0008303729118779302
131    ],
132    "p_value": 0.54535,
133    "mde": 0.0006149647377560715,
134    "mde_rel_pct": 27.80411594838706,
135    "verdict": "no significant win",
136    "system_worked": false
137  },
138  "mechanism_signature": {
139    "predicted_vs_observed": {
140      "n_test": 800,
141      "predicted_certificate_drop_mean": 0.3020721338689327,
142      "observed_certificate_drop_mean": -0.014734884549397975,
143      "predicted_vs_observed_drop_ratio": -0.048653354892595646,
144      "contract_permitted_fraction": 0.8,
145      "permitted_certificate_violations": 0,
146      "confirmed": 1.0
147    },
148    "confirmed": 1.0
149  }
150}