1{
  2  "bench_version": 1,
  3  "track": "fisher_multitoken_denoising",
  4  "model": "custom_denoiser",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "best_cfg": {
  9      "lr": 0.003,
 10      "smax": 1.0
 11    },
 12    "sweep": [
 13      {
 14        "cfg": {
 15          "lr": 0.001,
 16          "smax": 1.0
 17        },
 18        "mean": 0.4561383128166199
 19      },
 20      {
 21        "cfg": {
 22          "lr": 0.001,
 23          "smax": 2.0
 24        },
 25        "mean": 0.5842899084091187
 26      },
 27      {
 28        "cfg": {
 29          "lr": 0.003,
 30          "smax": 1.0
 31        },
 32        "mean": 0.3060583472251892
 33      },
 34      {
 35        "cfg": {
 36          "lr": 0.003,
 37          "smax": 2.0
 38        },
 39        "mean": 0.59112948179245
 40      },
 41      {
 42        "cfg": {
 43          "lr": 0.01,
 44          "smax": 1.0
 45        },
 46        "mean": 0.6318341046571732
 47      },
 48      {
 49        "cfg": {
 50          "lr": 0.01,
 51          "smax": 2.0
 52        },
 53        "mean": 0.883560985326767
 54      }
 55    ],
 56    "full": {
 57      "mean": 0.3756636306643486,
 58      "std": 0.0827197936263635,
 59      "per_seed": [
 60        0.29077786207199097,
 61        0.32696259021759033,
 62        0.3145223557949066,
 63        0.2919705808162689,
 64        0.4482685327529907,
 65        0.4811434745788574,
 66        0.5064208507537842,
 67        0.34524279832839966
 68      ],
 69      "n": 8
 70    }
 71  },
 72  "idea": {
 73    "mean": 0.38846133276820183,
 74    "std": 0.08522182674569097,
 75    "per_seed": [
 76      0.2787860035896301,
 77      0.3444952070713043,
 78      0.3325658440589905,
 79      0.30363890528678894,
 80      0.4642293453216553,
 81      0.5044667720794678,
 82      0.5081693530082703,
 83      0.37133923172950745
 84    ],
 85    "n": 8
 86  },
 87  "comparison": {
 88    "delta_mean": 0.012797702103853226,
 89    "idea_wins": 1,
 90    "n_pairs": 8,
 91    "per_seed_diffs": [
 92      -0.01199185848236084,
 93      0.01753261685371399,
 94      0.018043488264083862,
 95      0.01166832447052002,
 96      0.01596081256866455,
 97      0.02332329750061035,
 98      0.001748502254486084,
 99      0.026096433401107788
100    ],
101    "p_value": 0.03075,
102    "mde": 0.010412284911868961,
103    "mde_rel_pct": 2.77170427529947,
104    "verdict": "idea worse (significant)",
105    "system_worked": false
106  },
107  "mechanism_signature": {
108    "predicted_equal_arc_local_kl_cv": 1.7439286709141664e-06,
109    "observed_parameter_path_linear_kl_cv": 0.6206565234045026,
110    "trained_model_stage_behavior": [
111      {
112        "schedule": "linear",
113        "observed_stage_error_cv": 0.610444495224178,
114        "mean_stage_error": 0.08814288070425391
115      },
116      {
117        "schedule": "fisher",
118        "observed_stage_error_cv": 0.4789595501048241,
119        "mean_stage_error": 0.10801896243356168
120      }
121    ],
122    "confirmed": true,
123    "note": "Stage errors are measured on the trained Fisher model; analytic KL values are included only as the re-tested mechanism prediction."
124  },
125  "idea_sweep": [
126    {
127      "cfg": {
128        "lr": 0.003,
129        "smax": 1.0
130      },
131      "result": {
132        "mean": 0.38846133276820183,
133        "std": 0.08522182674569097,
134        "per_seed": [
135          0.2787860035896301,
136          0.3444952070713043,
137          0.3325658440589905,
138          0.30363890528678894,
139          0.4642293453216553,
140          0.5044667720794678,
141          0.5081693530082703,
142          0.37133923172950745
143        ],
144        "n": 8
145      }
146    },
147    {
148      "cfg": {
149        "lr": 0.001,
150        "smax": 1.0
151      },
152      "result": {
153        "mean": 0.49281759932637215,
154        "std": 0.03752558894150052,
155        "per_seed": [
156          0.47663557529449463,
157          0.5520936846733093,
158          0.5091399550437927,
159          0.48174160718917847,
160          0.4514659345149994,
161          0.48623156547546387,
162          0.544323742389679,
163          0.4409087300300598
164        ],
165        "n": 8
166      }
167    },
168    {
169      "cfg": {
170        "lr": 0.01,
171        "smax": 1.0
172      },
173      "result": {
174        "mean": 0.5914078056812286,
175        "std": 0.11582810816462852,
176        "per_seed": [
177          0.7036856412887573,
178          0.35683417320251465,
179          0.5790603756904602,
180          0.7571036219596863,
181          0.6265718936920166,
182          0.5940495133399963,
183          0.4918774962425232,
184          0.6220797300338745
185        ],
186        "n": 8
187      }
188    }
189  ],
190  "custom_track": {
191    "name": "fisher_multitoken_denoising",
192    "file": "fisher_bench_track.py",
193    "domain": "diffusion-sampling"
194  },
195  "structural_match": "multi-token denoising with correlated sequence distributions; schedule is the only method difference"
196}