1{
  2  "bench_version": 1,
  3  "track": "custom:bidirectional_conditional_compatibility",
  4  "model": "two_conditional_mlp",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "sweep": {
  9      "best_cfg": {
 10        "lr": 0.01
 11      },
 12      "sweep": [
 13        {
 14          "cfg": {
 15            "lr": 0.01
 16          },
 17          "mean": 1.326949805021286
 18        },
 19        {
 20          "cfg": {
 21            "lr": 0.03
 22          },
 23          "mean": 1.3306202590465546
 24        },
 25        {
 26          "cfg": {
 27            "lr": 0.1
 28          },
 29          "mean": 1.3483841717243195
 30        }
 31      ],
 32      "full": {
 33        "mean": 1.3114151656627655,
 34        "std": 0.02693886115100743,
 35        "per_seed": [
 36          1.329000473022461,
 37          1.3154711723327637,
 38          1.3227543830871582,
 39          1.3405731916427612,
 40          1.3417959213256836,
 41          1.3014893531799316,
 42          1.2759835720062256,
 43          1.2642532587051392
 44        ],
 45        "n": 8
 46      }
 47    },
 48    "best_config": {
 49      "lr": 0.01
 50    },
 51    "full": {
 52      "per_seed": [
 53        1.329000473022461,
 54        1.3154711723327637,
 55        1.3227543830871582,
 56        1.3405731916427612,
 57        1.3417959213256836,
 58        1.3014893531799316,
 59        1.2759835720062256,
 60        1.2642532587051392
 61      ],
 62      "mean": 1.3114151656627655,
 63      "std": 0.028798853947698433
 64    }
 65  },
 66  "idea": {
 67    "per_seed": [
 68      1.328962802886963,
 69      1.3155274391174316,
 70      1.3226888179779053,
 71      1.3404121398925781,
 72      1.341781735420227,
 73      1.3010740280151367,
 74      1.2759394645690918,
 75      1.2640597820281982
 76    ],
 77    "mean": 1.3113057762384415,
 78    "std": 0.028841388419379284
 79  },
 80  "comparison": {
 81    "delta_mean": -0.00010938942432403564,
 82    "idea_wins": 7,
 83    "n_pairs": 8,
 84    "per_seed_diffs": [
 85      -3.7670135498046875e-05,
 86      5.626678466796875e-05,
 87      -6.556510925292969e-05,
 88      -0.00016105175018310547,
 89      -1.4185905456542969e-05,
 90      -0.0004153251647949219,
 91      -4.410743713378906e-05,
 92      -0.00019347667694091797
 93    ],
 94    "p_value": 0.04555,
 95    "mde": 0.00012287599741212592,
 96    "mde_rel_pct": 0.009369725212078557,
 97    "verdict": "idea better (significant)",
 98    "system_worked": true
 99  },
100  "mechanism_signature": {
101    "custom_track": {
102      "name": "bidirectional_conditional_compatibility",
103      "file": "cycle_bench.py",
104      "domain": "paired categorical conditionals"
105    },
106    "prediction": "cycle penalty lowers held-out 95th-percentile |Delta| while retaining task NLL",
107    "baseline_heldout_p95": 0.06756381243467333,
108    "idea_heldout_p95": 0.036757445335388196,
109    "baseline_test_nll": 1.3114151656627655,
110    "idea_test_nll": 1.3113057762384415,
111    "observed_empirical_cycle_mean": 2.187573233669582e-07,
112    "confirmed": true
113  }
114}