1{
  2  "bench_version": 1,
  3  "track": "sequence",
  4  "model": "transformer_tiny",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "best_cfg": {
  9      "lr": 0.001
 10    },
 11    "sweep": [
 12      {
 13        "cfg": {
 14          "lr": 0.001
 15        },
 16        "mean": 0.620273545384407
 17      },
 18      {
 19        "cfg": {
 20          "lr": 0.002
 21        },
 22        "mean": 0.6989391446113586
 23      },
 24      {
 25        "cfg": {
 26          "lr": 0.003
 27        },
 28        "mean": 0.8049232810735703
 29      }
 30    ],
 31    "full": {
 32      "mean": 0.6231015399098396,
 33      "std": 0.06897174016713392,
 34      "per_seed": [
 35        0.7222549319267273,
 36        0.5303708910942078,
 37        0.5396761298179626,
 38        0.6887922286987305,
 39        0.640042245388031,
 40        0.6537070274353027,
 41        0.6618669033050537,
 42        0.5481019616127014
 43      ],
 44      "n": 8
 45    }
 46  },
 47  "idea": {
 48    "cfg": {
 49      "q": 0.5
 50    },
 51    "per_seed": [
 52      0.7222549319267273,
 53      0.5303708910942078,
 54      0.5396761298179626,
 55      0.6887922286987305,
 56      0.640042245388031,
 57      0.6537070274353027,
 58      0.6618669033050537,
 59      0.5481019616127014
 60    ],
 61    "mean": 0.6231015399098396
 62  },
 63  "comparison": {
 64    "delta_mean": 0.0,
 65    "idea_wins": 0,
 66    "n_pairs": 8,
 67    "per_seed_diffs": [
 68      0.0,
 69      0.0,
 70      0.0,
 71      0.0,
 72      0.0,
 73      0.0,
 74      0.0,
 75      0.0
 76    ],
 77    "p_value": 1.0,
 78    "mde": 0.0,
 79    "mde_rel_pct": 0.0,
 80    "verdict": "no measurable effect",
 81    "system_worked": false
 82  },
 83  "mathcheck": {
 84    "delta": 0.25,
 85    "capacity": 0.3475898813907971,
 86    "safe_capacity_eps_.1": 0.3128308932517174,
 87    "binary_stream_rate": 1.0,
 88    "rate_satisfies_bound": false,
 89    "transition_probability_definition": "P(flip)=q"
 90  },
 91  "idea_sweep": [
 92    {
 93      "cfg": {
 94        "q": 0.25
 95      },
 96      "per_seed": [
 97        0.712475597858429,
 98        0.5296003818511963,
 99        0.5759447813034058,
100        0.6928728818893433,
101        0.6395291686058044,
102        0.6558457016944885,
103        0.6588586568832397,
104        0.5424662828445435
105      ],
106      "mean": 0.6259491816163063
107    },
108    {
109      "cfg": {
110        "q": 0.4
111      },
112      "per_seed": [
113        0.7143200635910034,
114        0.5526124238967896,
115        0.5422033667564392,
116        0.6958776712417603,
117        0.625187873840332,
118        0.6721104383468628,
119        0.6582097411155701,
120        0.538433849811554
121      ],
122      "mean": 0.6248694285750389
123    },
124    {
125      "cfg": {
126        "q": 0.5
127      },
128      "per_seed": [
129        0.7222549319267273,
130        0.5303708910942078,
131        0.5396761298179626,
132        0.6887922286987305,
133        0.640042245388031,
134        0.6537070274353027,
135        0.6618669033050537,
136        0.5481019616127014
137      ],
138      "mean": 0.6231015399098396
139    }
140  ],
141  "protocol": {
142    "epochs": 8,
143    "batch": 128,
144    "train_test_n": 400,
145    "corruption": "25% random insertions in an 8-bit redundant stream"
146  },
147  "mechanism_signature": {
148    "predicted_markov_q": 0.5,
149    "observed_markov_transition_rate": 0.5178571428571428,
150    "predicted_iid_q": 0.5,
151    "observed_iid_transition_rate": 0.5178571428571428,
152    "trained_model_sensitivity_mean": {
153      "baseline": 0.024137018248438835,
154      "idea": 0.023845407529734075
155    },
156    "confirmed": true,
157    "per_seed": [
158      {
159        "seed": 0,
160        "baseline_stream_sensitivity": 0.017377333715558052,
161        "idea_stream_sensitivity": 0.01767236925661564
162      },
163      {
164        "seed": 1,
165        "baseline_stream_sensitivity": 0.026213817298412323,
166        "idea_stream_sensitivity": 0.02488153986632824
167      },
168      {
169        "seed": 2,
170        "baseline_stream_sensitivity": 0.023712292313575745,
171        "idea_stream_sensitivity": 0.02447468228638172
172      },
173      {
174        "seed": 3,
175        "baseline_stream_sensitivity": 0.009144331328570843,
176        "idea_stream_sensitivity": 0.0082431985065341
177      },
178      {
179        "seed": 4,
180        "baseline_stream_sensitivity": 0.01021712925285101,
181        "idea_stream_sensitivity": 0.010267045348882675
182      },
183      {
184        "seed": 5,
185        "baseline_stream_sensitivity": 0.06268110126256943,
186        "idea_stream_sensitivity": 0.06135376915335655
187      },
188      {
189        "seed": 6,
190        "baseline_stream_sensitivity": 0.01560310460627079,
191        "idea_stream_sensitivity": 0.017230955883860588
192      },
193      {
194        "seed": 7,
195        "baseline_stream_sensitivity": 0.028147036209702492,
196        "idea_stream_sensitivity": 0.026639699935913086
197      }
198    ]
199  }
200}