{
  "bench_version": 1,
  "track": "dynamics",
  "model": "rnn_small",
  "metric_direction": "lower is better",
  "n_seeds": 8,
  "baseline": {
    "best_cfg": {
      "lr": 0.006,
      "weight_decay": 0.0001
    },
    "sweep": [
      {
        "cfg": {
          "lr": 0.0015,
          "weight_decay": 0.0
        },
        "mean": 0.005481100233737379
      },
      {
        "cfg": {
          "lr": 0.0015,
          "weight_decay": 0.0001
        },
        "mean": 0.005474042263813317
      },
      {
        "cfg": {
          "lr": 0.003,
          "weight_decay": 0.0
        },
        "mean": 0.003975238680141047
      },
      {
        "cfg": {
          "lr": 0.003,
          "weight_decay": 0.0001
        },
        "mean": 0.00394076865632087
      },
      {
        "cfg": {
          "lr": 0.006,
          "weight_decay": 0.0
        },
        "mean": 0.002146739774616435
      },
      {
        "cfg": {
          "lr": 0.006,
          "weight_decay": 0.0001
        },
        "mean": 0.002088654466206208
      }
    ],
    "full": {
      "mean": 0.0022117759071988985,
      "std": 0.0004126881098400608,
      "per_seed": [
        0.002800536109134555,
        0.001347046927548945,
        0.001825663261115551,
        0.0023813715670257807,
        0.0023169058840721846,
        0.0024658460170030594,
        0.0022247303277254105,
        0.002332107163965702
      ],
      "n": 8
    }
  },
  "idea": {
    "mean": 0.0020463809778448194,
    "std": 0.0006329416993684799,
    "per_seed": [
      0.003160907654091716,
      0.0019281062996014953,
      0.0024306366685777903,
      0.0027447089087218046,
      0.0015351816546171904,
      0.0011941296979784966,
      0.0018756426870822906,
      0.001501734252087772
    ],
    "n": 8,
    "best_cfg": {
      "lr": 0.006,
      "weight_decay": 0.0001,
      "cert_weight": 0.01
    },
    "sweep": [
      {
        "cfg": {
          "lr": 0.006,
          "weight_decay": 0.0001,
          "cert_weight": 0.01
        },
        "mean": 0.0025660898827482015
      },
      {
        "cfg": {
          "lr": 0.006,
          "weight_decay": 0.0001,
          "cert_weight": 0.05
        },
        "mean": 0.0025660898827482015
      },
      {
        "cfg": {
          "lr": 0.006,
          "weight_decay": 0.0001,
          "cert_weight": 0.2
        },
        "mean": 0.0025660898827482015
      }
    ]
  },
  "comparison": {
    "delta_mean": -0.00016539492935407907,
    "idea_wins": 4,
    "n_pairs": 8,
    "per_seed_diffs": [
      0.00036037154495716095,
      0.0005810593720525503,
      0.0006049734074622393,
      0.00036333734169602394,
      -0.0007817242294549942,
      -0.0012717163190245628,
      -0.0003490876406431198,
      -0.0008303729118779302
    ],
    "p_value": 0.54535,
    "mde": 0.0006149647377560715,
    "mde_rel_pct": 27.80411594838706,
    "verdict": "no significant win",
    "system_worked": false
  },
  "mechanism_signature": {
    "predicted_vs_observed": {
      "n_test": 800,
      "predicted_certificate_drop_mean": 0.3020721338689327,
      "observed_certificate_drop_mean": -0.014734884549397975,
      "predicted_vs_observed_drop_ratio": -0.048653354892595646,
      "contract_permitted_fraction": 0.8,
      "permitted_certificate_violations": 0,
      "confirmed": 1.0
    },
    "confirmed": 1.0
  }
}