{
  "seed": 1071,
  "predictions": {
    "P1_pushforward_exact": "aggregation error is 0 up to floating point, and synonym splitting leaves state mass unchanged",
    "P2_power_distortion": "raw log-odds scale with gamma; fitted calibration slope a should satisfy gamma*a ~= 1",
    "P3_calibration_effect": "calibrated TV-to-exact-posterior should be near 0 for every gamma, while raw TV is 0 at gamma=1 and grows away from 1"
  },
  "sanity": {
    "pushforward_max_abs_error": 0.0,
    "synonym_split_max_abs_error": 1.1102230246251565e-16,
    "response_mass": 1.0,
    "pushed_mass": 1.0
  },
  "sweep": [
    {
      "gamma": 0.5,
      "predicted_a": 2.0,
      "fitted_a_mean": 1.8311318383591928,
      "gamma_times_fitted_a": 0.9155659191795964,
      "raw": {
        "nll": 0.8572267721071046,
        "brier": 0.5050365069825522,
        "tv_to_exact_q": 0.1369419752489948,
        "ece": 0.13228225310990974,
        "set90_coverage": 0.9986,
        "mean_set_size": 2.9038
      },
      "calibrated": {
        "nll": 0.8008695190016372,
        "brier": 0.4724195908791922,
        "tv_to_exact_q": 0.020023923130417648,
        "ece": 0.019039719148512373,
        "set90_coverage": 0.9754,
        "mean_set_size": 2.4738
      },
      "optimizer_success": false
    },
    {
      "gamma": 0.75,
      "predicted_a": 1.3333333333333333,
      "fitted_a_mean": 1.3110876536290874,
      "gamma_times_fitted_a": 0.9833157402218156,
      "raw": {
        "nll": 0.805427923492955,
        "brier": 0.4741506019639819,
        "tv_to_exact_q": 0.061074856125397876,
        "ece": 0.06562909937882927,
        "set90_coverage": 0.9872,
        "mean_set_size": 2.6366
      },
      "calibrated": {
        "nll": 0.7915430152128272,
        "brier": 0.46653125348977226,
        "tv_to_exact_q": 0.005265335286719836,
        "ece": 0.009886458166121609,
        "set90_coverage": 0.9702,
        "mean_set_size": 2.4068
      },
      "optimizer_success": true
    },
    {
      "gamma": 1.0,
      "predicted_a": 1.0,
      "fitted_a_mean": 0.9953532162760084,
      "gamma_times_fitted_a": 0.9953532162760084,
      "raw": {
        "nll": 0.7873009438915944,
        "brier": 0.4666413141016571,
        "tv_to_exact_q": 2.034366598790971e-17,
        "ece": 0.012771470251124017,
        "set90_coverage": 0.9734,
        "mean_set_size": 2.3876
      },
      "calibrated": {
        "nll": 0.7879509007202342,
        "brier": 0.46677964644116265,
        "tv_to_exact_q": 0.006602013699090465,
        "ece": 0.012340774685794783,
        "set90_coverage": 0.9734,
        "mean_set_size": 2.392
      },
      "optimizer_success": true
    },
    {
      "gamma": 1.5,
      "predicted_a": 0.6666666666666666,
      "fitted_a_mean": 0.6530733297958197,
      "gamma_times_fitted_a": 0.9796099946937296,
      "raw": {
        "nll": 0.8414786648391069,
        "brier": 0.48738748659195336,
        "tv_to_exact_q": 0.08651805518125863,
        "ece": 0.08809918856795493,
        "set90_coverage": 0.9234,
        "mean_set_size": 2.0218
      },
      "calibrated": {
        "nll": 0.8020212579724905,
        "brier": 0.47353656511301123,
        "tv_to_exact_q": 0.008298574265717416,
        "ece": 0.010963217301649434,
        "set90_coverage": 0.9718,
        "mean_set_size": 2.4056
      },
      "optimizer_success": true
    },
    {
      "gamma": 2.0,
      "predicted_a": 0.5,
      "fitted_a_mean": 0.5130615254222709,
      "gamma_times_fitted_a": 1.0261230508445418,
      "raw": {
        "nll": 0.889404327234238,
        "brier": 0.4903964225682859,
        "tv_to_exact_q": 0.14200148153616524,
        "ece": 0.1324196675129269,
        "set90_coverage": 0.8884,
        "mean_set_size": 1.778
      },
      "calibrated": {
        "nll": 0.7790706624494645,
        "brier": 0.4598805465689491,
        "tv_to_exact_q": 0.00697078504150675,
        "ece": 0.01752786632853536,
        "set90_coverage": 0.968,
        "mean_set_size": 2.357
      },
      "optimizer_success": false
    }
  ],
  "comparison_gamma_2": {
    "gamma": 2.0,
    "predicted_a": 0.5,
    "fitted_a_mean": 0.5130615254222709,
    "gamma_times_fitted_a": 1.0261230508445418,
    "raw": {
      "nll": 0.889404327234238,
      "brier": 0.4903964225682859,
      "tv_to_exact_q": 0.14200148153616524,
      "ece": 0.1324196675129269,
      "set90_coverage": 0.8884,
      "mean_set_size": 1.778
    },
    "calibrated": {
      "nll": 0.7790706624494645,
      "brier": 0.4598805465689491,
      "tv_to_exact_q": 0.00697078504150675,
      "ece": 0.01752786632853536,
      "set90_coverage": 0.968,
      "mean_set_size": 2.357
    },
    "optimizer_success": false
  }
}