1{
  2  "bench_version": 1,
  3  "track": "dynamics",
  4  "model": "rnn_small",
  5  "metric_direction": "lower is better",
  6  "n_seeds": 8,
  7  "baseline": {
  8    "best_cfg": {
  9      "lr": 0.006,
 10      "steps": 1
 11    },
 12    "sweep": [
 13      {
 14        "cfg": {
 15          "lr": 0.001,
 16          "steps": 1
 17        },
 18        "mean": 0.004103701037820429
 19      },
 20      {
 21        "cfg": {
 22          "lr": 0.001,
 23          "steps": 3
 24        },
 25        "mean": 0.005604122125077993
 26      },
 27      {
 28        "cfg": {
 29          "lr": 0.001,
 30          "steps": 5
 31        },
 32        "mean": 0.00564115314045921
 33      },
 34      {
 35        "cfg": {
 36          "lr": 0.003,
 37          "steps": 1
 38        },
 39        "mean": 0.0033861721749417484
 40      },
 41      {
 42        "cfg": {
 43          "lr": 0.003,
 44          "steps": 3
 45        },
 46        "mean": 0.005019772244850174
 47      },
 48      {
 49        "cfg": {
 50          "lr": 0.003,
 51          "steps": 5
 52        },
 53        "mean": 0.005090346734505147
 54      },
 55      {
 56        "cfg": {
 57          "lr": 0.006,
 58          "steps": 1
 59        },
 60        "mean": 0.0016983873210847378
 61      },
 62      {
 63        "cfg": {
 64          "lr": 0.006,
 65          "steps": 3
 66        },
 67        "mean": 0.00288440816802904
 68      },
 69      {
 70        "cfg": {
 71          "lr": 0.006,
 72          "steps": 5
 73        },
 74        "mean": 0.0029781226767227054
 75      }
 76    ],
 77    "full": {
 78      "mean": 0.0020863086683675647,
 79      "std": 0.00045500750701757853,
 80      "per_seed": [
 81        0.0018287990242242813,
 82        0.0014274335699155927,
 83        0.0014271392719820142,
 84        0.002110177418217063,
 85        0.002619170816615224,
 86        0.0024625908117741346,
 87        0.002196874236688018,
 88        0.00261828419752419
 89      ],
 90      "n": 8
 91    }
 92  },
 93  "idea": {
 94    "mean": 0.0035335189022589475,
 95    "std": 0.0006978225321427611,
 96    "per_seed": [
 97      0.0029865081887692213,
 98      0.002204712713137269,
 99      0.003266959683969617,
100      0.003441720036789775,
101      0.0042303199879825115,
102      0.004240280482918024,
103      0.0034899020101875067,
104      0.0044077481143176556
105    ],
106    "n": 8
107  },
108  "comparison": {
109    "delta_mean": 0.0014472102338913828,
110    "idea_wins": 0,
111    "n_pairs": 8,
112    "per_seed_diffs": [
113      0.00115770916454494,
114      0.0007772791432216763,
115      0.0018398204119876027,
116      0.001331542618572712,
117      0.0016111491713672876,
118      0.0017776896711438894,
119      0.0012930277734994888,
120      0.0017894639167934656
121    ],
122    "p_value": 0.0081,
123    "mde": 0.00031240367548733783,
124    "mde_rel_pct": 14.973991155957691,
125    "verdict": "idea worse (significant)",
126    "system_worked": false
127  },
128  "mechanism_signature": {
129    "math_check": {
130      "max_bellman_residual": 0.0,
131      "value_nonincreasing_with_cost": true,
132      "stop_boundary_nonincreasing_with_cost": true,
133      "costs": [
134        0,
135        0.02,
136        0.035,
137        0.04,
138        0.05
139      ],
140      "initial_values": [
141        0.19714903424072264,
142        0.09714903424072266,
143        0.02214903424072264,
144        0.0037074088745117084,
145        0.0
146      ],
147      "threshold_indices": [
148        20,
149        12,
150        4,
151        2,
152        0
153      ]
154    },
155    "state": "self-consistency verifier score from trained RNN successive refinement candidates",
156    "predicted_vs_observed": {
157      "predicted_continuation_decreases_with_cost": true,
158      "costs": [
159        0.0005,
160        0.0015,
161        0.003
162      ],
163      "observed_continue_fraction_by_cost": [
164        0.9968749955296516,
165        0.9940624907612801,
166        0.9871874675154686
167      ],
168      "observed_predicted_gain_by_cost": [
169        0.00294073588502215,
170        0.0030416790865274382,
171        0.003552038834823179
172      ],
173      "observed_task_mse_gain_by_cost": [
174        -9.425566531717778e-08,
175        -7.28626037016511e-07,
176        -1.4213641406968235e-06
177      ]
178    },
179    "confirmed": true
180  },
181  "idea_sweep": {
182    "best_cfg": {
183      "lr": 0.006,
184      "cost": 0.003
185    },
186    "sweep": [
187      {
188        "cfg": {
189          "lr": 0.006,
190          "cost": 0.0005
191        },
192        "mean": 0.0035419926862232387
193      },
194      {
195        "cfg": {
196          "lr": 0.006,
197          "cost": 0.0015
198        },
199        "mean": 0.003538758319336921
200      },
201      {
202        "cfg": {
203          "lr": 0.006,
204          "cost": 0.003
205        },
206        "mean": 0.0035335189022589475
207      }
208    ],
209    "full": {
210      "mean": 0.0035335189022589475,
211      "std": 0.0006978225321427611,
212      "per_seed": [
213        0.0029865081887692213,
214        0.002204712713137269,
215        0.003266959683969617,
216        0.003441720036789775,
217        0.0042303199879825115,
218        0.004240280482918024,
219        0.0034899020101875067,
220        0.0044077481143176556
221      ],
222      "n": 8
223    }
224  },
225  "protocol_notes": {
226    "structural_match": "dynamics: actuated pendulum rollout and sequential refinement",
227    "epochs": 12,
228    "n_train": 400,
229    "n_test": 400,
230    "paired_seeds": 8,
231    "baseline_knobs_swept": [
232      "lr",
233      "fixed refinement steps"
234    ],
235    "idea_knobs_swept": [
236      "lr",
237      "Bellman cost"
238    ]
239  }
240}