{
  "bench_version": 1,
  "track": "sequence",
  "model": "transformer_tiny",
  "metric_direction": "lower is better",
  "n_seeds": 8,
  "baseline": {
    "best_cfg": {
      "lr": 0.0015,
      "weight_decay": 0.0
    },
    "sweep": [
      {
        "cfg": {
          "lr": 0.0015,
          "weight_decay": 0.0
        },
        "mean": 0.5785842016339302
      },
      {
        "cfg": {
          "lr": 0.003,
          "weight_decay": 0.0
        },
        "mean": 0.6899479925632477
      },
      {
        "cfg": {
          "lr": 0.006,
          "weight_decay": 0.0
        },
        "mean": 0.8105297237634659
      }
    ],
    "full": {
      "mean": 0.562101598829031,
      "std": 0.08060035643973527,
      "per_seed": [
        0.6448022127151489,
        0.4093892276287079,
        0.5553309321403503,
        0.7048144340515137,
        0.5405545234680176,
        0.5604735016822815,
        0.5436524748802185,
        0.5377954840660095
      ],
      "n": 8
    }
  },
  "idea": {
    "mean": 0.5841716527938843,
    "std": 0.05076107251588834,
    "per_seed": [
      0.6083592772483826,
      0.5841273665428162,
      0.6231781840324402,
      0.5197466015815735,
      0.6223173141479492,
      0.5627732872962952,
      0.654932975769043,
      0.49793821573257446
    ],
    "n": 8
  },
  "comparison": {
    "delta_mean": 0.022070053964853287,
    "idea_wins": 3,
    "n_pairs": 8,
    "per_seed_diffs": [
      -0.03644293546676636,
      0.17473813891410828,
      0.06784725189208984,
      -0.18506783246994019,
      0.08176279067993164,
      0.002299785614013672,
      0.11128050088882446,
      -0.03985726833343506
    ],
    "p_value": 0.5933,
    "mde": 0.09347444437257611,
    "mde_rel_pct": 16.629457124352946,
    "verdict": "no significant win",
    "system_worked": false
  },
  "mechanism_signature": {
    "mechanism_signature": {
      "prediction": "identity insertion and O(eta) untied response",
      "confirmed": true,
      "idea_best_cfg": {
        "lr": 0.0015,
        "weight_decay": 0.0
      },
      "per_seed": [
        {
          "seed": 0,
          "insertion_relative_output_change": 1.742963640083417e-08,
          "trained_pair_residual": 0.013553540222346783,
          "untied_eta_response": [
            [
              1e-05,
              6.882298180244106e-07
            ],
            [
              0.0001,
              6.846436008345336e-06
            ],
            [
              0.001,
              6.854822277091444e-05
            ]
          ],
          "trained_loglog_slope": 0.9991313573267615
        },
        {
          "seed": 1,
          "insertion_relative_output_change": 1.768672674984373e-08,
          "trained_pair_residual": 0.01374770887196064,
          "untied_eta_response": [
            [
              1e-05,
              1.540886728434998e-06
            ],
            [
              0.0001,
              1.5476085536647588e-05
            ],
            [
              0.001,
              0.0001548135041957721
            ]
          ],
          "trained_loglog_slope": 1.0010190631783853
        },
        {
          "seed": 2,
          "insertion_relative_output_change": 1.7788378769978408e-08,
          "trained_pair_residual": 0.01247485727071762,
          "untied_eta_response": [
            [
              1e-05,
              2.696924639167264e-06
            ],
            [
              0.0001,
              2.6614063244778663e-05
            ],
            [
              0.001,
              0.0002659428573679179
            ]
          ],
          "trained_loglog_slope": 0.9969597598038572
        },
        {
          "seed": 3,
          "insertion_relative_output_change": 1.678153260797899e-08,
          "trained_pair_residual": 0.012198672629892826,
          "untied_eta_response": [
            [
              1e-05,
              3.872593197229435e-07
            ],
            [
              0.0001,
              3.2745517728471896e-06
            ],
            [
              0.001,
              3.2737443689256907e-05
            ]
          ],
          "trained_loglog_slope": 0.9635214430977415
        },
        {
          "seed": 4,
          "insertion_relative_output_change": 1.788907688649033e-08,
          "trained_pair_residual": 0.012925044633448124,
          "untied_eta_response": [
            [
              1e-05,
              1.7712729913910152e-06
            ],
            [
              0.0001,
              1.768083166098222e-05
            ],
            [
              0.001,
              0.00017684148042462766
            ]
          ],
          "trained_loglog_slope": 0.9996493207179143
        },
        {
          "seed": 5,
          "insertion_relative_output_change": 1.6752988329926666e-08,
          "trained_pair_residual": 0.011602357029914856,
          "untied_eta_response": [
            [
              1e-05,
              9.838264531936147e-07
            ],
            [
              0.0001,
              9.362133823742624e-06
            ],
            [
              0.001,
              9.372592467116192e-05
            ]
          ],
          "trained_loglog_slope": 0.9894706190051281
        },
        {
          "seed": 6,
          "insertion_relative_output_change": 1.6823253901065982e-08,
          "trained_pair_residual": 0.012010437436401844,
          "untied_eta_response": [
            [
              1e-05,
              1.4330839803733397e-06
            ],
            [
              0.0001,
              1.4343587281473447e-05
            ],
            [
              0.001,
              0.00014369039854500443
            ]
          ],
          "trained_loglog_slope": 1.0005780540284896
        },
        {
          "seed": 7,
          "insertion_relative_output_change": 1.7179964117985946e-08,
          "trained_pair_residual": 0.013055686838924885,
          "untied_eta_response": [
            [
              1e-05,
              6.872157314319338e-07
            ],
            [
              0.0001,
              6.427855623769574e-06
            ],
            [
              0.001,
              6.434921670006588e-05
            ]
          ],
          "trained_loglog_slope": 0.9857250860758257
        },
        {
          "seed": 0,
          "insertion_relative_output_change": 1.742963640083417e-08,
          "trained_pair_residual": 0.024646366015076637,
          "untied_eta_response": [
            [
              1e-05,
              6.151723823677457e-07
            ],
            [
              0.0001,
              6.120769739936804e-06
            ],
            [
              0.001,
              6.104174099164084e-05
            ]
          ],
          "trained_loglog_slope": 0.9983150409119986
        },
        {
          "seed": 1,
          "insertion_relative_output_change": 1.768672674984373e-08,
          "trained_pair_residual": 0.026445627212524414,
          "untied_eta_response": [
            [
              1e-05,
              1.0641194876370719e-06
            ],
            [
              0.0001,
              1.0895923878706526e-05
            ],
            [
              0.001,
              0.00010886218660743907
            ]
          ],
          "trained_loglog_slope": 1.0049433283222116
        },
        {
          "seed": 2,
          "insertion_relative_output_change": 1.7788378769978408e-08,
          "trained_pair_residual": 0.023685336112976074,
          "untied_eta_response": [
            [
              1e-05,
              1.0162499393118196e-06
            ],
            [
              0.0001,
              1.0251549610984512e-05
            ],
            [
              0.001,
              0.00010272490908391774
            ]
          ],
          "trained_loglog_slope": 1.0023376164486255
        },
        {
          "seed": 3,
          "insertion_relative_output_change": 1.678153260797899e-08,
          "trained_pair_residual": 0.024265628308057785,
          "untied_eta_response": [
            [
              1e-05,
              2.682033937162487e-06
            ],
            [
              0.0001,
              2.6792338758241385e-05
            ],
            [
              0.001,
              0.0002678318414837122
            ]
          ],
          "trained_loglog_slope": 0.9996989692102766
        },
        {
          "seed": 4,
          "insertion_relative_output_change": 1.788907688649033e-08,
          "trained_pair_residual": 0.02498849853873253,
          "untied_eta_response": [
            [
              1e-05,
              1.6889165408429108e-06
            ],
            [
              0.0001,
              1.6884097931324504e-05
            ],
            [
              0.001,
              0.00016868698003236204
            ]
          ],
          "trained_loglog_slope": 0.9997366870611765
        },
        {
          "seed": 5,
          "insertion_relative_output_change": 1.6752988329926666e-08,
          "trained_pair_residual": 0.02313663437962532,
          "untied_eta_response": [
            [
              1e-05,
              1.1504967005748767e-06
            ],
            [
              0.0001,
              1.1409663784434088e-05
            ],
            [
              0.001,
              0.00011402922245906666
            ]
          ],
          "trained_loglog_slope": 0.9980653927231617
        },
        {
          "seed": 6,
          "insertion_relative_output_change": 1.6823253901065982e-08,
          "trained_pair_residual": 0.023537755012512207,
          "untied_eta_response": [
            [
              1e-05,
              1.739130652822496e-06
            ],
            [
              0.0001,
              1.758110556693282e-05
            ],
            [
              0.001,
              0.00017618776473682374
            ]
          ],
          "trained_loglog_slope": 1.0028217680103706
        },
        {
          "seed": 7,
          "insertion_relative_output_change": 1.7179964117985946e-08,
          "trained_pair_residual": 0.024072429165244102,
          "untied_eta_response": [
            [
              1e-05,
              7.772475214551378e-07
            ],
            [
              0.0001,
              7.599023319926346e-06
            ],
            [
              0.001,
              7.574163464596495e-05
            ]
          ],
          "trained_loglog_slope": 0.9943876639826758
        },
        {
          "seed": 0,
          "insertion_relative_output_change": 1.742963640083417e-08,
          "trained_pair_residual": 0.049147915095090866,
          "untied_eta_response": [
            [
              1e-05,
              1.9303267890791176e-06
            ],
            [
              0.0001,
              1.8857945178751834e-05
            ],
            [
              0.001,
              0.0001881620701169595
            ]
          ],
          "trained_loglog_slope": 0.9944506222877422
        },
        {
          "seed": 1,
          "insertion_relative_output_change": 1.768672674984373e-08,
          "trained_pair_residual": 0.04954760521650314,
          "untied_eta_response": [
            [
              1e-05,
              8.311401870741975e-06
            ],
            [
              0.0001,
              8.277293818537146e-05
            ],
            [
              0.001,
              0.0008280269103124738
            ]
          ],
          "trained_loglog_slope": 0.9991850847890872
        },
        {
          "seed": 2,
          "insertion_relative_output_change": 1.7788378769978408e-08,
          "trained_pair_residual": 0.04762578755617142,
          "untied_eta_response": [
            [
              1e-05,
              7.512046067859046e-06
            ],
            [
              0.0001,
              7.488950359402224e-05
            ],
            [
              0.001,
              0.0007490166462957859
            ]
          ],
          "trained_loglog_slope": 0.9993666135316805
        },
        {
          "seed": 3,
          "insertion_relative_output_change": 1.678153260797899e-08,
          "trained_pair_residual": 0.045538078993558884,
          "untied_eta_response": [
            [
              1e-05,
              1.7467951693106443e-05
            ],
            [
              0.0001,
              0.000174284607055597
            ],
            [
              0.001,
              0.0017429785802960396
            ]
          ],
          "trained_loglog_slope": 0.9995250339052046
        },
        {
          "seed": 4,
          "insertion_relative_output_change": 1.788907688649033e-08,
          "trained_pair_residual": 0.056724853813648224,
          "untied_eta_response": [
            [
              1e-05,
              4.201423053018516e-06
            ],
            [
              0.0001,
              4.1662235162220895e-05
            ],
            [
              0.001,
              0.00041710384539328516
            ]
          ],
          "trained_loglog_slope": 0.9984238898394366
        },
        {
          "seed": 5,
          "insertion_relative_output_change": 1.6752988329926666e-08,
          "trained_pair_residual": 0.046788010746240616,
          "untied_eta_response": [
            [
              1e-05,
              5.517295448953519e-06
            ],
            [
              0.0001,
              5.510038317879662e-05
            ],
            [
              0.001,
              0.0005517865065485239
            ]
          ],
          "trained_loglog_slope": 1.000022417554908
        },
        {
          "seed": 6,
          "insertion_relative_output_change": 1.6823253901065982e-08,
          "trained_pair_residual": 0.04568944126367569,
          "untied_eta_response": [
            [
              1e-05,
              5.431155841506552e-06
            ],
            [
              0.0001,
              5.4161326261237264e-05
            ],
            [
              0.001,
              0.0005412694299593568
            ]
          ],
          "trained_loglog_slope": 0.9992606174539198
        },
        {
          "seed": 7,
          "insertion_relative_output_change": 1.7179964117985946e-08,
          "trained_pair_residual": 0.04901719093322754,
          "untied_eta_response": [
            [
              1e-05,
              5.4369570534618106e-06
            ],
            [
              0.0001,
              5.419322042143904e-05
            ],
            [
              0.001,
              0.0005417013308033347
            ]
          ],
          "trained_loglog_slope": 0.9992020000011609
        }
      ],
      "mean_insertion_relative_output_change": 1.7291444720513027e-08,
      "mean_trained_pair_residual": 0.028601060970686376
    }
  }
}