1{
  2  "bench_version": 1,
  3  "track": "dynamics",
  4  "model": "cyclic_rnn",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "best_cfg": {
  9      "lr": 0.01,
 10      "weight_decay": 0.0
 11    },
 12    "sweep": [
 13      {
 14        "cfg": {
 15          "lr": 0.001,
 16          "weight_decay": 0.0
 17        },
 18        "mean": 0.073138820938766
 19      },
 20      {
 21        "cfg": {
 22          "lr": 0.003,
 23          "weight_decay": 0.0
 24        },
 25        "mean": 0.01337075512856245
 26      },
 27      {
 28        "cfg": {
 29          "lr": 0.01,
 30          "weight_decay": 0.0
 31        },
 32        "mean": 0.004462164011783898
 33      }
 34    ],
 35    "full": {
 36      "mean": 0.004087189445272088,
 37      "std": 0.0010007874399045344,
 38      "per_seed": [
 39        0.005726089235395193,
 40        0.0033913287334144115,
 41        0.004283653572201729,
 42        0.004447584506124258,
 43        0.0049942112527787685,
 44        0.0028478815220296383,
 45        0.002599592786282301,
 46        0.004407173953950405
 47      ],
 48      "n": 8
 49    }
 50  },
 51  "idea": {
 52    "mean": 0.004080623359186575,
 53    "std": 0.0010004881538153208,
 54    "per_seed": [
 55      0.005722619593143463,
 56      0.003379821078851819,
 57      0.004280869383364916,
 58      0.004445591010153294,
 59      0.004983154591172934,
 60      0.0028435608837753534,
 61      0.0025958793703466654,
 62      0.0043934909626841545
 63    ],
 64    "n": 8
 65  },
 66  "comparison": {
 67    "delta_mean": -6.566086085513234e-06,
 68    "idea_wins": 8,
 69    "n_pairs": 8,
 70    "per_seed_diffs": [
 71      -3.469642251729965e-06,
 72      -1.1507654562592506e-05,
 73      -2.784188836812973e-06,
 74      -1.9934959709644318e-06,
 75      -1.1056661605834961e-05,
 76      -4.320638254284859e-06,
 77      -3.7134159356355667e-06,
 78      -1.368299126625061e-05
 79    ],
 80    "p_value": 0.0081,
 81    "mde": 3.91188445758878e-06,
 82    "mde_rel_pct": 0.0957108670877467,
 83    "verdict": "idea better (significant)",
 84    "system_worked": true
 85  },
 86  "mechanism_signature": {
 87    "note": "trained-model zero-input Fourier perturbation; q=0",
 88    "signature": {
 89      "q": 0,
 90      "kappa": 0.15,
 91      "rows": [
 92        {
 93          "system": "baseline",
 94          "weights": [
 95            1.1289324760437012,
 96            1.1723648309707642,
 97            1.1778165102005005,
 98            1.2012455463409424
 99          ],
100          "modes": [
101            {
102              "mode": 1,
103              "predicted": -0.09919014415880456,
104              "observed": -0.04295965945097118
105            },
106            {
107              "mode": 2,
108              "predicted": -0.3536465652791994,
109              "observed": -0.17734158165416888
110            },
111            {
112              "mode": 3,
113              "predicted": -0.6558346787734254,
114              "observed": -0.42319134910019884
115            }
116          ]
117        },
118        {
119          "system": "idea",
120          "weights": [
121            1.1295150518417358,
122            1.173497200012207,
123            1.1802256107330322,
124            1.20441734790802
125          ],
126          "modes": [
127            {
128              "mode": 1,
129              "predicted": -0.09940500360509097,
130              "observed": -0.04295041987070078
131            },
132            {
133              "mode": 2,
134              "predicted": -0.3544018135383647,
135              "observed": -0.17730516519047812
136            },
137            {
138              "mode": 3,
139              "predicted": -0.6571983150619246,
140              "observed": -0.42311798056693856
141            }
142          ]
143        }
144      ],
145      "max_relative_error": 0.5679249704438308,
146      "confirmed": false,
147      "task_metric_seed0": {
148        "baseline": 0.005726089235395193,
149        "idea": 0.005722619593143463
150      }
151    }
152  },
153  "idea_sweep": [
154    {
155      "cfg": {
156        "lr": 0.001,
157        "weight_decay": 0.0
158      },
159      "mean": 0.07314319256693125
160    },
161    {
162      "cfg": {
163        "lr": 0.003,
164        "weight_decay": 0.0
165      },
166      "mean": 0.013368312735110521
167    },
168    {
169      "cfg": {
170        "lr": 0.01,
171        "weight_decay": 0.0
172      },
173      "mean": 0.004457225266378373
174    }
175  ],
176  "protocol": {
177    "paired_seeds": [
178      0,
179      1,
180      2,
181      3,
182      4,
183      5,
184      6,
185      7
186    ],
187    "train_samples": 400,
188    "test_samples": 200,
189    "epochs": 18,
190    "architecture": "same CyclicRNN; barrier only differs"
191  }
192}