Стенд-проверка (stage-2) · промт оператора:

(универсальный)

Ответ агента:

{ "worked": false, "confidence": 10, "verdict": "Implemented a fractional exponential-memory recurrent model and evaluated it against a GRU on the structurally matched dynamics track using 8 paired seeds, equal epochs and batch size, and parity-preserving learning-rate sweeps. The baseline achieved mean MSE 1.477801822602487e-05 versus 2.5887351512210444e-04 for the idea; paired delta was +2.4409549689607957e-04 with permutation p=0.0081, so the idea was significantly worse. Stability held numerically, but the trained-bank power-law slope was -0.8731883349875018 versus the predicted -0.5, so the mechanism signature was not confirmed.", "metrics": { "baseline": "GRU, best lr=0.01; 8-seed mean MSE 1.477801822602487e-05, std 5.904456722576056e-06.", "idea": "8-mode fractional recurrent bank, p=0.5, best lr=0.003; 8-seed mean MSE 2.5887351512210444e-04, std 5.818967880653584e-05.", "paired_delta": "2.4409549689607957e-04, idea minus baseline; positive is worse.", "permutation_p_value": 0.0081, "idea_wins": 0, "n_pairs": 8, "mechanism_signature": { "predicted_log_slope": -0.5, "observed_trained_model_log_slope": -0.8731883349875018, "absolute_error": 0.3731883349875018, "confirmed": false, "stability_spectral_radii": [ 0.9992190550963239, 0.9922179382602435, 0.9692332344763441 ] } }, "how_to_run": "/home/maxwelhelp/main/bin/python3 bench_fractional.py", "files": [ "bench_fractional.py", "bench_report.json", "rerun_output.txt" ], "limitations": "Only the built-in dynamics track was tested; sequence forecasting and long-context extrapolation were not tested. The fractional model uses a custom recurrent transition rather than an exact internal GRU ablation, and its rates and weights were fixed rather than learned. The run used 12 epochs and three learning rates, within the small-model benchmark budget.", "bench_report": { "bench_version": 1, "track": "dynamics", "model": "rnn_small_fractional", "metric_direction": "lower is better", "n_seeds": 8, "baseline": { "best_cfg": { "lr": 0.01 }, "sweep": [ { "cfg": { "lr": 0.001 }, "mean": 0.0004224302538204938 }, { "cfg": { "lr": 0.003 }, "mean": 3.919086066161981e-05 }, { "cfg": { "lr": 0.01 }, "mean": 1.4552034826920135e-05 } ], "full": { "mean": 1.477801822602487e-05, "std": 5.904456722576056e-06, "per_seed": [ 2.641624269017484e-05, 8.83757365954807e-06, 8.367352165805642e-06, 1.4586970792151988e-05, 1.0999674486811273e-05, 2.1751329768449068e-05, 1.276554939977359e-05, 1.4499452845484484e-05 ], "n": 8 } }, "idea": { "best_cfg": { "lr": 0.003, "p": 0.5 }, "mean": 0.00025887351512210444, "std": 5.818967880653584e-05, "per_seed": [ 0.0002734987938310951, 0.00015879524289630353, 0.00034040064201690257, 0.00028941663913428783, 0.000296872021863237, 0.00026502739638090134, 0.0002749341365415603, 0.0001720432483125478 ], "n": 8 }, "comparison": { "delta_mean": 0.00024409549689607957, "idea_wins": 0, "n_pairs": 8, "per_seed_diffs": [ 0.00024708255114092026, 0.00014995766923675546, 0.0003320332898510969, 0.00027482966834213585, 0.00028587234737642575, 0.00024327606661245227, 0.0002621685871417867, 0.00015754379546706332 ], "p_value": 0.0081, "mde": 5.2037973707415915e-05, "mde_rel_pct": 352.13093468631877, "verdict": "idea worse (significant)", "system_worked": false }, "stage2_protocol": { "epochs": 12, "batch": 128, "baseline_grid": [ { "lr": 0.001 }, { "lr": 0.003 }, { "lr": 0.01 } ], "idea_grid": [ { "lr": 0.001, "p": 0.5 }, { "lr": 0.003, "p": 0.5 }, { "lr": 0.01, "p": 0.5 } ], "paired_seeds": [ 0, 1, 2, 3, 4, 5, 6, 7 ] } }, "system_verdict": "failed", "practical_verdict": "harms", "mechanism_ok": null, "system_judged": true }