ISS-Constrained Modular Recurrent Network / bench_report.json

Failed on benchmark

Raw ⬇ ZIP
  1{
  2  "bench_version": 1,
  3  "track": "dynamics",
  4  "model": "rnn_small",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "best_cfg": {
  9      "lr": 0.003
 10    },
 11    "sweep": [
 12      {
 13        "cfg": {
 14          "lr": 0.0015
 15        },
 16        "mean": 0.01389938301872462
 17      },
 18      {
 19        "cfg": {
 20          "lr": 0.003
 21        },
 22        "mean": 0.006244247197173536
 23      },
 24      {
 25        "cfg": {
 26          "lr": 0.006
 27        },
 28        "mean": 0.007728174212388694
 29      }
 30    ],
 31    "full": {
 32      "mean": 0.0067647010437212884,
 33      "std": 0.0039439644249273965,
 34      "per_seed": [
 35        0.00582656916230917,
 36        0.0031395666301250458,
 37        0.012371733784675598,
 38        0.0036391192115843296,
 39        0.002855701372027397,
 40        0.00469561293721199,
 41        0.007864921353757381,
 42        0.013724383898079395
 43      ],
 44      "n": 8
 45    }
 46  },
 47  "idea": {
 48    "mean": 0.013474244042299688,
 49    "std": 0.005054578092032932,
 50    "per_seed": [
 51      0.007837503217160702,
 52      0.024782177060842514,
 53      0.007902047596871853,
 54      0.0145634850487113,
 55      0.013514268212020397,
 56      0.010362541303038597,
 57      0.015141943469643593,
 58      0.013689986430108547
 59    ],
 60    "n": 8
 61  },
 62  "comparison": {
 63    "delta_mean": 0.006709542998578399,
 64    "idea_wins": 2,
 65    "n_pairs": 8,
 66    "per_seed_diffs": [
 67      0.002010934054851532,
 68      0.021642610430717468,
 69      -0.004469686187803745,
 70      0.01092436583712697,
 71      0.010658566839993,
 72      0.005666928365826607,
 73      0.007277022115886211,
 74      -3.4397467970848083e-05
 75    ],
 76    "p_value": 0.04685,
 77    "mde": 0.00670636373721558,
 78    "mde_rel_pct": 99.13762180872938,
 79    "verdict": "idea worse (significant)",
 80    "system_worked": false
 81  },
 82  "mechanism_signature": {
 83    "protocol_note": "Complete 8-seed paired protocol; reduced to 400 train/400 test and 8 epochs because standard 4000/30 run exceeded environment time budget.",
 84    "prediction": "ISS residual cognitive perturbations remain contractive under the conservative bound",
 85    "signature": {
 86      "quantity": "trained perturbation norm ratio over 8 steps",
 87      "baseline_observed_ratio": 1.3777021169662476,
 88      "idea_observed_ratio": 1.9764628410339355,
 89      "idea_conservative_predicted_bound": 0.9299999999999999,
 90      "confirmed": false,
 91      "note": "GRU signature uses final hidden perturbation; modular uses cognitive trajectory."
 92    },
 93    "idea_sweep": [
 94      {
 95        "cfg": {
 96          "lr": 0.003,
 97          "gamma": 1.5,
 98          "dt": 0.1
 99        },
100        "mean": 0.013771303230896592
101      },
102      {
103        "cfg": {
104          "lr": 0.003,
105          "gamma": 2.0,
106          "dt": 0.1
107        },
108        "mean": 0.01809797785244882
109      },
110      {
111        "cfg": {
112          "lr": 0.003,
113          "gamma": 2.5,
114          "dt": 0.1
115        },
116        "mean": 0.02604409120976925
117      }
118    ],
119    "parameter_counts": {
120      "baseline": 13313,
121      "idea": 6689
122    }
123  }
124}