1{
  2  "bench_version": 1,
  3  "track": "conditional_multitoken_diffusion",
  4  "model": "mlp_tiny",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "best_cfg": {
  9      "lr": 0.003
 10    },
 11    "sweep": [
 12      {
 13        "cfg": {
 14          "lr": 0.001
 15        },
 16        "mean": 0.08654761128127575
 17      },
 18      {
 19        "cfg": {
 20          "lr": 0.003
 21        },
 22        "mean": 0.07295109704136848
 23      },
 24      {
 25        "cfg": {
 26          "lr": 0.01
 27        },
 28        "mean": 0.07489615492522717
 29      }
 30    ],
 31    "full": {
 32      "mean": 0.07604722864925861,
 33      "std": 0.007765201697800962,
 34      "per_seed": [
 35        0.06751493364572525,
 36        0.06798635423183441,
 37        0.08183492720127106,
 38        0.07446817308664322,
 39        0.08383433520793915,
 40        0.0655355304479599,
 41        0.08018559962511063,
 42        0.0870179757475853
 43      ],
 44      "n": 8
 45    }
 46  },
 47  "idea": {
 48    "mean": 0.07605153322219849,
 49    "std": 0.007456295188304263,
 50    "per_seed": [
 51      0.06727362424135208,
 52      0.06813433766365051,
 53      0.08250146359205246,
 54      0.07412734627723694,
 55      0.0823955237865448,
 56      0.06638552248477936,
 57      0.08205603063106537,
 58      0.08553841710090637
 59    ],
 60    "n": 8
 61  },
 62  "comparison": {
 63    "delta_mean": 4.304572939872742e-06,
 64    "idea_wins": 4,
 65    "n_pairs": 8,
 66    "per_seed_diffs": [
 67      -0.00024130940437316895,
 68      0.00014798343181610107,
 69      0.0006665363907814026,
 70      -0.0003408268094062805,
 71      -0.0014388114213943481,
 72      0.000849992036819458,
 73      0.0018704310059547424,
 74      -0.0014795586466789246
 75    ],
 76    "p_value": 1.0,
 77    "mde": 0.0009544449208548851,
 78    "mde_rel_pct": 1.2550686432728932,
 79    "verdict": "no measurable effect",
 80    "system_worked": false
 81  },
 82  "mechanism_signature": {
 83    "idea_sweep": [
 84      {
 85        "cfg": {
 86          "lr": 0.001
 87        },
 88        "mean": 0.08863587398082018,
 89        "std": 0.005865804558271554
 90      },
 91      {
 92        "cfg": {
 93          "lr": 0.003
 94        },
 95        "mean": 0.07605153322219849,
 96        "std": 0.007456295188304263
 97      },
 98      {
 99        "cfg": {
100          "lr": 0.01
101        },
102        "mean": 0.0774571718648076,
103        "std": 0.0066361820117857954
104      }
105    ],
106    "selected_idea_cfg": {
107      "lr": 0.003
108    },
109    "mechanism_signature": {
110      "prediction": "scrambled Sobol batch-mean variance lower and slope closer to -1 than IID",
111      "trained_model_batch_mean_sd": {
112        "iid": 0.0008616268068651571,
113        "sobol": 0.0003355427185340906
114      },
115      "observed_log2N_slopes": {
116        "iid": -0.6601498584773617,
117        "sobol": -0.4193229684723195
118      },
119      "variance_ratio_sobol_over_iid": 0.3894292933560067,
120      "confirmed": false
121    },
122    "custom_track": {
123      "name": "conditional_multitoken_diffusion",
124      "file": "bench/custom_tracks/conditional_multitoken_diffusion.py",
125      "domain": "diffusion-sampling"
126    }
127  }
128}