1{
  2  "seed": 1071,
  3  "predictions": {
  4    "P1_pushforward_exact": "aggregation error is 0 up to floating point, and synonym splitting leaves state mass unchanged",
  5    "P2_power_distortion": "raw log-odds scale with gamma; fitted calibration slope a should satisfy gamma*a ~= 1",
  6    "P3_calibration_effect": "calibrated TV-to-exact-posterior should be near 0 for every gamma, while raw TV is 0 at gamma=1 and grows away from 1"
  7  },
  8  "sanity": {
  9    "pushforward_max_abs_error": 0.0,
 10    "synonym_split_max_abs_error": 1.1102230246251565e-16,
 11    "response_mass": 1.0,
 12    "pushed_mass": 1.0
 13  },
 14  "sweep": [
 15    {
 16      "gamma": 0.5,
 17      "predicted_a": 2.0,
 18      "fitted_a_mean": 1.8311318383591928,
 19      "gamma_times_fitted_a": 0.9155659191795964,
 20      "raw": {
 21        "nll": 0.8572267721071046,
 22        "brier": 0.5050365069825522,
 23        "tv_to_exact_q": 0.1369419752489948,
 24        "ece": 0.13228225310990974,
 25        "set90_coverage": 0.9986,
 26        "mean_set_size": 2.9038
 27      },
 28      "calibrated": {
 29        "nll": 0.8008695190016372,
 30        "brier": 0.4724195908791922,
 31        "tv_to_exact_q": 0.020023923130417648,
 32        "ece": 0.019039719148512373,
 33        "set90_coverage": 0.9754,
 34        "mean_set_size": 2.4738
 35      },
 36      "optimizer_success": false
 37    },
 38    {
 39      "gamma": 0.75,
 40      "predicted_a": 1.3333333333333333,
 41      "fitted_a_mean": 1.3110876536290874,
 42      "gamma_times_fitted_a": 0.9833157402218156,
 43      "raw": {
 44        "nll": 0.805427923492955,
 45        "brier": 0.4741506019639819,
 46        "tv_to_exact_q": 0.061074856125397876,
 47        "ece": 0.06562909937882927,
 48        "set90_coverage": 0.9872,
 49        "mean_set_size": 2.6366
 50      },
 51      "calibrated": {
 52        "nll": 0.7915430152128272,
 53        "brier": 0.46653125348977226,
 54        "tv_to_exact_q": 0.005265335286719836,
 55        "ece": 0.009886458166121609,
 56        "set90_coverage": 0.9702,
 57        "mean_set_size": 2.4068
 58      },
 59      "optimizer_success": true
 60    },
 61    {
 62      "gamma": 1.0,
 63      "predicted_a": 1.0,
 64      "fitted_a_mean": 0.9953532162760084,
 65      "gamma_times_fitted_a": 0.9953532162760084,
 66      "raw": {
 67        "nll": 0.7873009438915944,
 68        "brier": 0.4666413141016571,
 69        "tv_to_exact_q": 2.034366598790971e-17,
 70        "ece": 0.012771470251124017,
 71        "set90_coverage": 0.9734,
 72        "mean_set_size": 2.3876
 73      },
 74      "calibrated": {
 75        "nll": 0.7879509007202342,
 76        "brier": 0.46677964644116265,
 77        "tv_to_exact_q": 0.006602013699090465,
 78        "ece": 0.012340774685794783,
 79        "set90_coverage": 0.9734,
 80        "mean_set_size": 2.392
 81      },
 82      "optimizer_success": true
 83    },
 84    {
 85      "gamma": 1.5,
 86      "predicted_a": 0.6666666666666666,
 87      "fitted_a_mean": 0.6530733297958197,
 88      "gamma_times_fitted_a": 0.9796099946937296,
 89      "raw": {
 90        "nll": 0.8414786648391069,
 91        "brier": 0.48738748659195336,
 92        "tv_to_exact_q": 0.08651805518125863,
 93        "ece": 0.08809918856795493,
 94        "set90_coverage": 0.9234,
 95        "mean_set_size": 2.0218
 96      },
 97      "calibrated": {
 98        "nll": 0.8020212579724905,
 99        "brier": 0.47353656511301123,
100        "tv_to_exact_q": 0.008298574265717416,
101        "ece": 0.010963217301649434,
102        "set90_coverage": 0.9718,
103        "mean_set_size": 2.4056
104      },
105      "optimizer_success": true
106    },
107    {
108      "gamma": 2.0,
109      "predicted_a": 0.5,
110      "fitted_a_mean": 0.5130615254222709,
111      "gamma_times_fitted_a": 1.0261230508445418,
112      "raw": {
113        "nll": 0.889404327234238,
114        "brier": 0.4903964225682859,
115        "tv_to_exact_q": 0.14200148153616524,
116        "ece": 0.1324196675129269,
117        "set90_coverage": 0.8884,
118        "mean_set_size": 1.778
119      },
120      "calibrated": {
121        "nll": 0.7790706624494645,
122        "brier": 0.4598805465689491,
123        "tv_to_exact_q": 0.00697078504150675,
124        "ece": 0.01752786632853536,
125        "set90_coverage": 0.968,
126        "mean_set_size": 2.357
127      },
128      "optimizer_success": false
129    }
130  ],
131  "comparison_gamma_2": {
132    "gamma": 2.0,
133    "predicted_a": 0.5,
134    "fitted_a_mean": 0.5130615254222709,
135    "gamma_times_fitted_a": 1.0261230508445418,
136    "raw": {
137      "nll": 0.889404327234238,
138      "brier": 0.4903964225682859,
139      "tv_to_exact_q": 0.14200148153616524,
140      "ece": 0.1324196675129269,
141      "set90_coverage": 0.8884,
142      "mean_set_size": 1.778
143    },
144    "calibrated": {
145      "nll": 0.7790706624494645,
146      "brier": 0.4598805465689491,
147      "tv_to_exact_q": 0.00697078504150675,
148      "ece": 0.01752786632853536,
149      "set90_coverage": 0.968,
150      "mean_set_size": 2.357
151    },
152    "optimizer_success": false
153  }
154}