Saturation-Adaptive Prefill Chunking / bench_report.json

Failed on benchmark

Raw ⬇ ZIP
  1{
  2  "bench_version": 1,
  3  "track": "dynamics",
  4  "model": "rnn_small",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "best_cfg": {
  9      "lr": 0.01
 10    },
 11    "sweep": [
 12      {
 13        "cfg": {
 14          "lr": 0.001
 15        },
 16        "mean": 0.023484921315684915
 17      },
 18      {
 19        "cfg": {
 20          "lr": 0.003
 21        },
 22        "mean": 0.006196767906658351
 23      },
 24      {
 25        "cfg": {
 26          "lr": 0.01
 27        },
 28        "mean": 0.0053210142941679806
 29      }
 30    ],
 31    "full": {
 32      "mean": 0.004259285735315643,
 33      "std": 0.002730084338516835,
 34      "per_seed": [
 35        0.0023857334163039923,
 36        0.00896634440869093,
 37        0.0018012352520599961,
 38        0.008130744099617004,
 39        0.003042455529794097,
 40        0.0010564422700554132,
 41        0.0035787634551525116,
 42        0.005112567450851202
 43      ],
 44      "n": 8
 45    }
 46  },
 47  "idea": {
 48    "mean": 0.0030400985560845584,
 49    "std": 0.00126101653222839,
 50    "per_seed": [
 51      0.0022289918269962072,
 52      0.002270024036988616,
 53      0.001566944643855095,
 54      0.0029011231381446123,
 55      0.005441730376332998,
 56      0.003352733328938484,
 57      0.0019949846900999546,
 58      0.004564256407320499
 59    ],
 60    "n": 8
 61  },
 62  "comparison": {
 63    "delta_mean": -0.0012191871792310849,
 64    "idea_wins": 6,
 65    "n_pairs": 8,
 66    "per_seed_diffs": [
 67      -0.00015674158930778503,
 68      -0.006696320371702313,
 69      -0.00023429060820490122,
 70      -0.005229620961472392,
 71      0.0023992748465389013,
 72      0.002296291058883071,
 73      -0.001583778765052557,
 74      -0.0005483110435307026
 75    ],
 76    "p_value": 0.36405,
 77    "mde": 0.002722617253919733,
 78    "mde_rel_pct": 63.921920789330834,
 79    "verdict": "no significant win",
 80    "system_worked": false
 81  },
 82  "mechanism_signature": {
 83    "prediction": "adaptive chunking lowers high-load sequential-state ramp while preserving task function",
 84    "observed_baseline_state_ramp": 0.029594257241114974,
 85    "observed_adaptive_state_ramp": 0.041038555558770895,
 86    "relative_reduction": -0.38670672571422005,
 87    "confirmed": false,
 88    "note": "activation ramp is a hardware-independent proxy; no NVML power available"
 89  },
 90  "custom_track": null,
 91  "idea_sweep": {
 92    "best_cfg": {
 93      "lr": 0.01
 94    },
 95    "sweep": [
 96      {
 97        "cfg": {
 98          "lr": 0.001
 99        },
100        "mean": 0.024194482597522438
101      },
102      {
103        "cfg": {
104          "lr": 0.003
105        },
106        "mean": 0.0037008895887993276
107      },
108      {
109        "cfg": {
110          "lr": 0.01
111        },
112        "mean": 0.0022417709114961326
113      }
114    ],
115    "full": {
116      "mean": 0.0030400985560845584,
117      "std": 0.00126101653222839,
118      "per_seed": [
119        0.0022289918269962072,
120        0.002270024036988616,
121        0.001566944643855095,
122        0.0029011231381446123,
123        0.005441730376332998,
124        0.003352733328938484,
125        0.0019949846900999546,
126        0.004564256407320499
127      ],
128      "n": 8
129    }
130  },
131  "idea_lr": 0.01,
132  "paired_baseline_at_idea_lr": [
133    0.0023857334163039923,
134    0.00896634440869093,
135    0.0018012352520599961,
136    0.008130744099617004,
137    0.003042455529794097,
138    0.0010564422700554132,
139    0.0035787634551525116,
140    0.005112567450851202
141  ]
142}