{
  "bench_version": 1,
  "track": "custom:bidirectional_conditional_compatibility",
  "model": "two_conditional_mlp",
  "metric_direction": "lower is better",
  "n_seeds": 8,
  "baseline": {
    "sweep": {
      "best_cfg": {
        "lr": 0.01
      },
      "sweep": [
        {
          "cfg": {
            "lr": 0.01
          },
          "mean": 1.326949805021286
        },
        {
          "cfg": {
            "lr": 0.03
          },
          "mean": 1.3306202590465546
        },
        {
          "cfg": {
            "lr": 0.1
          },
          "mean": 1.3483841717243195
        }
      ],
      "full": {
        "mean": 1.3114151656627655,
        "std": 0.02693886115100743,
        "per_seed": [
          1.329000473022461,
          1.3154711723327637,
          1.3227543830871582,
          1.3405731916427612,
          1.3417959213256836,
          1.3014893531799316,
          1.2759835720062256,
          1.2642532587051392
        ],
        "n": 8
      }
    },
    "best_config": {
      "lr": 0.01
    },
    "full": {
      "per_seed": [
        1.329000473022461,
        1.3154711723327637,
        1.3227543830871582,
        1.3405731916427612,
        1.3417959213256836,
        1.3014893531799316,
        1.2759835720062256,
        1.2642532587051392
      ],
      "mean": 1.3114151656627655,
      "std": 0.028798853947698433
    }
  },
  "idea": {
    "per_seed": [
      1.328962802886963,
      1.3155274391174316,
      1.3226888179779053,
      1.3404121398925781,
      1.341781735420227,
      1.3010740280151367,
      1.2759394645690918,
      1.2640597820281982
    ],
    "mean": 1.3113057762384415,
    "std": 0.028841388419379284
  },
  "comparison": {
    "delta_mean": -0.00010938942432403564,
    "idea_wins": 7,
    "n_pairs": 8,
    "per_seed_diffs": [
      -3.7670135498046875e-05,
      5.626678466796875e-05,
      -6.556510925292969e-05,
      -0.00016105175018310547,
      -1.4185905456542969e-05,
      -0.0004153251647949219,
      -4.410743713378906e-05,
      -0.00019347667694091797
    ],
    "p_value": 0.04555,
    "mde": 0.00012287599741212592,
    "mde_rel_pct": 0.009369725212078557,
    "verdict": "idea better (significant)",
    "system_worked": true
  },
  "mechanism_signature": {
    "custom_track": {
      "name": "bidirectional_conditional_compatibility",
      "file": "cycle_bench.py",
      "domain": "paired categorical conditionals"
    },
    "prediction": "cycle penalty lowers held-out 95th-percentile |Delta| while retaining task NLL",
    "baseline_heldout_p95": 0.06756381243467333,
    "idea_heldout_p95": 0.036757445335388196,
    "baseline_test_nll": 1.3114151656627655,
    "idea_test_nll": 1.3113057762384415,
    "observed_empirical_cycle_mean": 2.187573233669582e-07,
    "confirmed": true
  }
}