{
  "bench_version": 1,
  "track": "graph_landmark_forecast",
  "model": "transformer_tiny",
  "metric_direction": "lower is better",
  "n_seeds": 8,
  "baseline": {
    "best_cfg": {
      "lr": 0.003
    },
    "sweep": [
      {
        "cfg": {
          "lr": 0.001
        },
        "mean": 0.09760121069848537
      },
      {
        "cfg": {
          "lr": 0.003
        },
        "mean": 0.09715960919857025
      },
      {
        "cfg": {
          "lr": 0.01
        },
        "mean": 0.09787633270025253
      }
    ],
    "full": {
      "mean": 0.09728806372731924,
      "std": 0.0028138072522287676,
      "per_seed": [
        0.0970306470990181,
        0.09655177593231201,
        0.09898804873228073,
        0.09606796503067017,
        0.10214304178953171,
        0.09961570054292679,
        0.09206581115722656,
        0.09584151953458786
      ],
      "n": 8
    },
    "full_grid": [
      {
        "cfg": {
          "lr": 0.001
        },
        "result": {
          "mean": 0.09813422616571188,
          "std": 0.003308826432344093,
          "per_seed": [
            0.09822424501180649,
            0.09727376699447632,
            0.0995187908411026,
            0.09538803994655609,
            0.10483681410551071,
            0.10019651055335999,
            0.09305645525455475,
            0.0965791866183281
          ],
          "n": 8
        }
      },
      {
        "cfg": {
          "lr": 0.003
        },
        "result": {
          "mean": 0.09728806372731924,
          "std": 0.0028138072522287676,
          "per_seed": [
            0.0970306470990181,
            0.09655177593231201,
            0.09898804873228073,
            0.09606796503067017,
            0.10214304178953171,
            0.09961570054292679,
            0.09206581115722656,
            0.09584151953458786
          ],
          "n": 8
        }
      },
      {
        "cfg": {
          "lr": 0.01
        },
        "result": {
          "mean": 0.09821864496916533,
          "std": 0.002263958786368283,
          "per_seed": [
            0.09782364219427109,
            0.09844150394201279,
            0.09902690351009369,
            0.09621328115463257,
            0.10131707042455673,
            0.10156738013029099,
            0.09458042681217194,
            0.09677895158529282
          ],
          "n": 8
        }
      }
    ]
  },
  "idea": {
    "mean": 0.06668837741017342,
    "std": 0.00810202264282473,
    "per_seed": [
      0.06714659929275513,
      0.07387638092041016,
      0.06558588147163391,
      0.059248507022857666,
      0.05166328698396683,
      0.06404716521501541,
      0.07355235517024994,
      0.07838684320449829
    ],
    "n": 8
  },
  "comparison": {
    "delta_mean": -0.030599686317145824,
    "idea_wins": 8,
    "n_pairs": 8,
    "per_seed_diffs": [
      -0.02988404780626297,
      -0.022675395011901855,
      -0.03340216726064682,
      -0.0368194580078125,
      -0.05047975480556488,
      -0.03556853532791138,
      -0.018513455986976624,
      -0.01745467633008957
    ],
    "p_value": 0.0081,
    "mde": 0.00920648763417379,
    "mde_rel_pct": 9.463121457507778,
    "verdict": "idea better (significant)",
    "system_worked": true
  },
  "custom_track": {
    "name": "graph_landmark_forecast",
    "file": "graph_landmark_track.py",
    "domain": "graph-nn"
  },
  "idea_config": {
    "lr": 0.01
  },
  "idea_sweep": [
    {
      "cfg": {
        "lr": 0.001
      },
      "result": {
        "mean": 0.0952951293438673,
        "std": 0.0034853121866759325,
        "per_seed": [
          0.0953633040189743,
          0.09373979270458221,
          0.09905679523944855,
          0.092075414955616,
          0.10213538259267807,
          0.0953550785779953,
          0.09400807321071625,
          0.09062719345092773
        ],
        "n": 8
      }
    },
    {
      "cfg": {
        "lr": 0.003
      },
      "result": {
        "mean": 0.08661147300153971,
        "std": 0.004209441400448157,
        "per_seed": [
          0.08820009231567383,
          0.09083075076341629,
          0.0886211097240448,
          0.08095572143793106,
          0.09157451242208481,
          0.07902113348245621,
          0.0880790650844574,
          0.08560939878225327
        ],
        "n": 8
      }
    },
    {
      "cfg": {
        "lr": 0.01
      },
      "result": {
        "mean": 0.06668837741017342,
        "std": 0.00810202264282473,
        "per_seed": [
          0.06714659929275513,
          0.07387638092041016,
          0.06558588147163391,
          0.059248507022857666,
          0.05166328698396683,
          0.06404716521501541,
          0.07355235517024994,
          0.07838684320449829
        ],
        "n": 8
      }
    }
  ],
  "mechanism_signature": {
    "prediction": "expected collisions are bounded by the hypergeometric union bound",
    "c_min": 1,
    "s": 8,
    "predicted_collision_upper_bound": 114.0,
    "observed_idea_collision_count": 0.0,
    "baseline_collision_count": 19.0,
    "trained_model_observations": [
      {
        "seed": 0,
        "baseline_metric": 0.0970306470990181,
        "idea_metric": 0.08820009231567383,
        "baseline_collisions": 19,
        "idea_collisions": 0,
        "idea_pred_target_corr": 0.892614061575404
      },
      {
        "seed": 1,
        "baseline_metric": 0.09655177593231201,
        "idea_metric": 0.09083075076341629,
        "baseline_collisions": 19,
        "idea_collisions": 0,
        "idea_pred_target_corr": 0.886642980654682
      },
      {
        "seed": 2,
        "baseline_metric": 0.09898804873228073,
        "idea_metric": 0.0886211097240448,
        "baseline_collisions": 19,
        "idea_collisions": 0,
        "idea_pred_target_corr": 0.8899489630055591
      },
      {
        "seed": 3,
        "baseline_metric": 0.09606796503067017,
        "idea_metric": 0.08095572143793106,
        "baseline_collisions": 19,
        "idea_collisions": 0,
        "idea_pred_target_corr": 0.8968686834235331
      },
      {
        "seed": 4,
        "baseline_metric": 0.10214304178953171,
        "idea_metric": 0.09157451242208481,
        "baseline_collisions": 19,
        "idea_collisions": 0,
        "idea_pred_target_corr": 0.8833831997989209
      },
      {
        "seed": 5,
        "baseline_metric": 0.09961570054292679,
        "idea_metric": 0.07902113348245621,
        "baseline_collisions": 19,
        "idea_collisions": 0,
        "idea_pred_target_corr": 0.8995094696825414
      },
      {
        "seed": 6,
        "baseline_metric": 0.09206581115722656,
        "idea_metric": 0.0880790650844574,
        "baseline_collisions": 19,
        "idea_collisions": 0,
        "idea_pred_target_corr": 0.8922238907368641
      },
      {
        "seed": 7,
        "baseline_metric": 0.09584151953458786,
        "idea_metric": 0.08560939878225327,
        "baseline_collisions": 19,
        "idea_collisions": 0,
        "idea_pred_target_corr": 0.8906957622074757
      }
    ],
    "confirmed": true
  }
}