Geometrically Attracting Random Recurrent Layer / bench_report.json

Failed on benchmark

Raw ⬇ ZIP
  1{
  2  "bench_version": 1,
  3  "track": "dynamics",
  4  "model": "rnn_small",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "best_cfg": {
  9      "lr": 0.01
 10    },
 11    "sweep": [
 12      {
 13        "cfg": {
 14          "lr": 0.001
 15        },
 16        "mean": 0.0038340488099493086
 17      },
 18      {
 19        "cfg": {
 20          "lr": 0.003
 21        },
 22        "mean": 0.000746978163078893
 23      },
 24      {
 25        "cfg": {
 26          "lr": 0.01
 27        },
 28        "mean": 9.810126357479021e-05
 29      }
 30    ],
 31    "full": {
 32      "mean": 0.0001305075584241422,
 33      "std": 5.472800825829195e-05,
 34      "per_seed": [
 35        0.00010314843530068174,
 36        6.60803125356324e-05,
 37        0.00012244189565535635,
 38        0.00010073441080749035,
 39        9.771589247975498e-05,
 40        0.00018152409757021815,
 41        0.0001220697449753061,
 42        0.00025034567806869745
 43      ],
 44      "n": 8
 45    }
 46  },
 47  "idea": {
 48    "mean": 0.00015589705117236008,
 49    "std": 7.121662686835463e-05,
 50    "per_seed": [
 51      0.00011286333028692752,
 52      5.95987839915324e-05,
 53      0.00010499265044927597,
 54      0.00014471430040430278,
 55      0.00017019343795254827,
 56      0.00019453300046734512,
 57      0.00014643977920059115,
 58      0.00031384112662635744
 59    ],
 60    "n": 8
 61  },
 62  "comparison": {
 63    "delta_mean": 2.538949274821789e-05,
 64    "idea_wins": 2,
 65    "n_pairs": 8,
 66    "per_seed_diffs": [
 67      9.714894986245781e-06,
 68      -6.4815285441000015e-06,
 69      -1.7449245206080377e-05,
 70      4.397988959681243e-05,
 71      7.247754547279328e-05,
 72      1.3008902897126973e-05,
 73      2.4370034225285053e-05,
 74      6.349544855765998e-05
 75    ],
 76    "p_value": 0.07125,
 77    "mde": 2.6938989951374075e-05,
 78    "mde_rel_pct": 20.64170863102341,
 79    "verdict": "no significant win",
 80    "system_worked": false
 81  },
 82  "mechanism_signature": {
 83    "predicted_vs_observed": {
 84      "predicted_log_expected_gain": -0.005753826349973679,
 85      "observed_log_distance_slope": -0.3882254362106326,
 86      "relative_slope_error": 66.47245187333293,
 87      "mean_route_probabilities": [
 88        0.5307331085205078,
 89        0.4692668914794922
 90      ],
 91      "candidate_spectral_gains": [
 92        0.8478590250015259,
 93        1.1598429679870605
 94      ],
 95      "test_mse": 0.00011286333028692752,
 96      "confirmed": false
 97    },
 98    "idea_sweep": [
 99      {
100        "cfg": {
101          "lr": 0.001
102        },
103        "result": {
104          "mean": 0.004973278002580628,
105          "std": 0.0010229290410082186,
106          "per_seed": [
107            0.0032259898725897074,
108            0.005356050096452236,
109            0.004283194895833731,
110            0.005885367747396231,
111            0.004350616130977869,
112            0.0045798118226230145,
113            0.005348666105419397,
114            0.006756527349352837
115          ],
116          "n": 8
117        }
118      },
119      {
120        "cfg": {
121          "lr": 0.003
122        },
123        "result": {
124          "mean": 0.0016119294887175784,
125          "std": 0.000517161965727792,
126          "per_seed": [
127            0.0012174114817753434,
128            0.002363392151892185,
129            0.0014342936920002103,
130            0.002530658384785056,
131            0.0011475831270217896,
132            0.0010538878850638866,
133            0.0014910241588950157,
134            0.0016571850283071399
135          ],
136          "n": 8
137        }
138      },
139      {
140        "cfg": {
141          "lr": 0.01
142        },
143        "result": {
144          "mean": 0.00015589705117236008,
145          "std": 7.121662686835463e-05,
146          "per_seed": [
147            0.00011286333028692752,
148            5.95987839915324e-05,
149            0.00010499265044927597,
150            0.00014471430040430278,
151            0.00017019343795254827,
152            0.00019453300046734512,
153            0.00014643977920059115,
154            0.00031384112662635744
155          ],
156          "n": 8
157        }
158      }
159    ],
160    "track_justification": "Dynamics is the built-in structural match for recurrent stability/control."
161  },
162  "protocol": {
163    "epochs": 12,
164    "n_train": 1200,
165    "n_test": 400,
166    "lr_union": [
167      0.001,
168      0.003,
169      0.01
170    ],
171    "idea_lambda": 0.05,
172    "target_rho": 0.95
173  }
174}