{
  "bench_version": 1,
  "track": "sequence",
  "model": "transformer_tiny",
  "metric_direction": "lower is better",
  "n_seeds": 8,
  "baseline": {
    "best_cfg": {
      "lr": 0.001
    },
    "sweep": [
      {
        "cfg": {
          "lr": 0.001
        },
        "mean": 0.620273545384407
      },
      {
        "cfg": {
          "lr": 0.002
        },
        "mean": 0.6989391446113586
      },
      {
        "cfg": {
          "lr": 0.003
        },
        "mean": 0.8049232810735703
      }
    ],
    "full": {
      "mean": 0.6231015399098396,
      "std": 0.06897174016713392,
      "per_seed": [
        0.7222549319267273,
        0.5303708910942078,
        0.5396761298179626,
        0.6887922286987305,
        0.640042245388031,
        0.6537070274353027,
        0.6618669033050537,
        0.5481019616127014
      ],
      "n": 8
    }
  },
  "idea": {
    "cfg": {
      "q": 0.5
    },
    "per_seed": [
      0.7222549319267273,
      0.5303708910942078,
      0.5396761298179626,
      0.6887922286987305,
      0.640042245388031,
      0.6537070274353027,
      0.6618669033050537,
      0.5481019616127014
    ],
    "mean": 0.6231015399098396
  },
  "comparison": {
    "delta_mean": 0.0,
    "idea_wins": 0,
    "n_pairs": 8,
    "per_seed_diffs": [
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0
    ],
    "p_value": 1.0,
    "mde": 0.0,
    "mde_rel_pct": 0.0,
    "verdict": "no measurable effect",
    "system_worked": false
  },
  "mathcheck": {
    "delta": 0.25,
    "capacity": 0.3475898813907971,
    "safe_capacity_eps_.1": 0.3128308932517174,
    "binary_stream_rate": 1.0,
    "rate_satisfies_bound": false,
    "transition_probability_definition": "P(flip)=q"
  },
  "idea_sweep": [
    {
      "cfg": {
        "q": 0.25
      },
      "per_seed": [
        0.712475597858429,
        0.5296003818511963,
        0.5759447813034058,
        0.6928728818893433,
        0.6395291686058044,
        0.6558457016944885,
        0.6588586568832397,
        0.5424662828445435
      ],
      "mean": 0.6259491816163063
    },
    {
      "cfg": {
        "q": 0.4
      },
      "per_seed": [
        0.7143200635910034,
        0.5526124238967896,
        0.5422033667564392,
        0.6958776712417603,
        0.625187873840332,
        0.6721104383468628,
        0.6582097411155701,
        0.538433849811554
      ],
      "mean": 0.6248694285750389
    },
    {
      "cfg": {
        "q": 0.5
      },
      "per_seed": [
        0.7222549319267273,
        0.5303708910942078,
        0.5396761298179626,
        0.6887922286987305,
        0.640042245388031,
        0.6537070274353027,
        0.6618669033050537,
        0.5481019616127014
      ],
      "mean": 0.6231015399098396
    }
  ],
  "protocol": {
    "epochs": 8,
    "batch": 128,
    "train_test_n": 400,
    "corruption": "25% random insertions in an 8-bit redundant stream"
  },
  "mechanism_signature": {
    "predicted_markov_q": 0.5,
    "observed_markov_transition_rate": 0.5178571428571428,
    "predicted_iid_q": 0.5,
    "observed_iid_transition_rate": 0.5178571428571428,
    "trained_model_sensitivity_mean": {
      "baseline": 0.024137018248438835,
      "idea": 0.023845407529734075
    },
    "confirmed": true,
    "per_seed": [
      {
        "seed": 0,
        "baseline_stream_sensitivity": 0.017377333715558052,
        "idea_stream_sensitivity": 0.01767236925661564
      },
      {
        "seed": 1,
        "baseline_stream_sensitivity": 0.026213817298412323,
        "idea_stream_sensitivity": 0.02488153986632824
      },
      {
        "seed": 2,
        "baseline_stream_sensitivity": 0.023712292313575745,
        "idea_stream_sensitivity": 0.02447468228638172
      },
      {
        "seed": 3,
        "baseline_stream_sensitivity": 0.009144331328570843,
        "idea_stream_sensitivity": 0.0082431985065341
      },
      {
        "seed": 4,
        "baseline_stream_sensitivity": 0.01021712925285101,
        "idea_stream_sensitivity": 0.010267045348882675
      },
      {
        "seed": 5,
        "baseline_stream_sensitivity": 0.06268110126256943,
        "idea_stream_sensitivity": 0.06135376915335655
      },
      {
        "seed": 6,
        "baseline_stream_sensitivity": 0.01560310460627079,
        "idea_stream_sensitivity": 0.017230955883860588
      },
      {
        "seed": 7,
        "baseline_stream_sensitivity": 0.028147036209702492,
        "idea_stream_sensitivity": 0.026639699935913086
      }
    ]
  }
}