Saturation-Adaptive Prefill Chunking / results.json

Failed on benchmark

Raw ⬇ ZIP
  1{
  2  "seed": 1663,
  3  "epsilon": 0.05,
  4  "target": 20.0,
  5  "controller_check": {
  6    "interior_slope_observed": -28.000000000000004,
  7    "interior_slope_predicted": -28.0,
  8    "s_monotone": true,
  9    "w_monotone": true,
 10    "clipped_range": [
 11      36.0,
 12      64.0
 13    ]
 14  },
 15  "main_comparison": {
 16    "fixed_chunk": 64.0,
 17    "adaptive_mean_chunk": 42.13646407352229,
 18    "adaptive_min_max_chunk": [
 19      22.35268117792745,
 20      54.45948665314698
 21    ],
 22    "fixed_ramp_q95": 28.73084473270458,
 23    "adaptive_ramp_q95": 15.582595671477533,
 24    "ramp_reduction_percent": 45.76353108845518,
 25    "fixed_peak_q99": 560.3831789238642,
 26    "adaptive_peak_q99": 543.6840767604164,
 27    "peak_change_percent": -2.9799435085678505,
 28    "throughput_proxy_fixed_tokens_per_quantum": 64.0,
 29    "throughput_proxy_adaptive_tokens_per_quantum": 42.13646407352229,
 30    "throughput_proxy_change_percent": -34.16177488512142,
 31    "latency_proxy_quantum_ratio_adaptive_over_fixed": 1.5188744809799148
 32  },
 33  "chunk_sweep": {
 34    "points": [
 35      {
 36        "chunk": 8,
 37        "ramp_q95": 15.879802706657847,
 38        "peak_q99": 544.3918872212083
 39      },
 40      {
 41        "chunk": 16,
 42        "ramp_q95": 17.769874489415418,
 43        "peak_q99": 548.0870382748652
 44      },
 45      {
 46        "chunk": 32,
 47        "ramp_q95": 21.56489902886325,
 48        "peak_q99": 555.483543979908
 49      },
 50      {
 51        "chunk": 48,
 52        "ramp_q95": 25.345112857324175,
 53        "peak_q99": 562.8485149973421
 54      },
 55      {
 56        "chunk": 64,
 57        "ramp_q95": 29.136761428933212,
 58        "peak_q99": 570.2387080082889
 59      }
 60    ],
 61    "q95_ramp_slope_per_token": 0.23673332404473343,
 62    "predicted_sign": "positive"
 63  },
 64  "regime_sweep": [
 65    {
 66      "saturation": 0.15,
 67      "whale_fraction": 0.0,
 68      "adaptive_mean_chunk": 59.78863645048558,
 69      "fixed_ramp_q95": 6.911981588947738,
 70      "adaptive_ramp_q95": 6.318632253560602,
 71      "reduction_percent": 8.584359315075464
 72    },
 73    {
 74      "saturation": 0.15,
 75      "whale_fraction": 0.5,
 76      "adaptive_mean_chunk": 49.33350585814084,
 77      "fixed_ramp_q95": 29.16603147738406,
 78      "adaptive_ramp_q95": 20.23008559826713,
 79      "reduction_percent": 30.63819596452827
 80    },
 81    {
 82      "saturation": 0.5,
 83      "whale_fraction": 0.0,
 84      "adaptive_mean_chunk": 49.99344781375241,
 85      "fixed_ramp_q95": 6.8722876642096065,
 86      "adaptive_ramp_q95": 5.814223242443008,
 87      "reduction_percent": 15.396102047312777
 88    },
 89    {
 90      "saturation": 0.5,
 91      "whale_fraction": 0.5,
 92      "adaptive_mean_chunk": 39.382356770270775,
 93      "fixed_ramp_q95": 29.117263536288363,
 94      "adaptive_ramp_q95": 16.45297122569248,
 95      "reduction_percent": 43.494102029239755
 96    },
 97    {
 98      "saturation": 0.85,
 99      "whale_fraction": 0.0,
100      "adaptive_mean_chunk": 40.20714051306561,
101      "fixed_ramp_q95": 7.142982267685329,
102      "adaptive_ramp_q95": 5.487084022618661,
103      "reduction_percent": 23.182169337839596
104    },
105    {
106      "saturation": 0.85,
107      "whale_fraction": 0.5,
108      "adaptive_mean_chunk": 29.627122439923717,
109      "fixed_ramp_q95": 29.11593154092076,
110      "adaptive_ramp_q95": 12.702679307224129,
111      "reduction_percent": 56.37206630544777
112    }
113  ],
114  "quantile_selection": {
115    "target_ramp": 20.0,
116    "largest_feasible_chunk": 16,
117    "candidate_reserves": {
118      "8": 15.879802706657847,
119      "16": 17.769874489415418,
120      "32": 21.56489902886325,
121      "48": 25.345112857324175,
122      "64": 29.136761428933212
123    }
124  },
125  "predictions_observed_vs_predicted": {
126    "P1_controller_slope_vs_formula": {
127      "predicted": -28.0,
128      "observed": -28.000000000000004,
129      "units": "tokens per saturation unit"
130    },
131    "P2_ramp_reserve_chunk_slope_positive": {
132      "predicted": "> 0",
133      "observed": 0.23673332404473343,
134      "units": "q95 ramp units per token"
135    },
136    "P3_high_load_reduction_exceeds_low_load": {
137      "predicted": "high > low",
138      "observed_high_percent": 56.37206630544777,
139      "observed_low_percent": 8.584359315075464
140    },
141    "P4_peak_change_is_small": {
142      "predicted": "absolute change < 5%",
143      "observed_percent": -2.9799435085678505
144    }
145  },
146  "checks": {
147    "controller_formula_pass": true,
148    "ramp_increases_with_chunk": true,
149    "high_load_benefit_exceeds_low_load": true,
150    "peak_change_small_main_case": true
151  },
152  "all_mechanism_checks_pass": true
153}