Стенд-проверка (stage-2) · промт оператора:

(универсальный)

Ответ агента:

{ "worked": false, "confidence": 9, "verdict": "The registered custom track stochastic_intervention_scm was accepted and evaluated through the official bench APIs. The joint model achieved mean MSE 2.2138 versus the tuned baseline's 2.4147, but the paired delta of -0.2009 had p=0.41835, so the official verdict is no significant win. The trained-model mechanism signature was confirmed, with selected-mediator prediction error 0.0646 for the idea versus 0.1293 for baseline.", "metrics": { "baseline": "mlp_tiny, tuned lr=0.003; sweep means lr=0.003: 2.4976, lr=0.01: 2.5648, lr=0.03: 2.6781; full 8-seed mean MSE 2.4147", "idea": "Joint model, best lr=0.03; sweep means lr=0.003: 7.9196, lr=0.01: 3.1929, lr=0.03: 2.2138; full 8-seed mean MSE 2.2138", "comparison": "delta_mean=-0.2009, idea_wins=6/8, permutation p_value=0.41835, verdict=no significant win", "mechanism_signature": "baseline selected-M absolute error=0.1293; idea selected-M absolute error=0.0646; confirmed=true" }, "bench_report": { "bench_version": 1, "track": "stochastic_intervention_scm", "model": "mlp_tiny", "metric_direction": "lower is better", "n_seeds": 8, "baseline": { "best_cfg": { "lr": 0.003 }, "sweep": [ { "cfg": { "lr": 0.003 }, "mean": 2.497592270374298 }, { "cfg": { "lr": 0.01 }, "mean": 2.5648475885391235 }, { "cfg": { "lr": 0.03 }, "mean": 2.6781041622161865 } ], "full": { "mean": 2.414719134569168, "std": 0.23689432610640407, "per_seed": [ 2.342486619949341, 2.4945199489593506, 2.7365739345550537, 2.4167885780334473, 2.2174108028411865, 2.058196783065796, 2.258479356765747, 2.793297052383423 ], "n": 8 } }, "idea": { "mean": 2.2138066589832306, "std": 0.5331661449712295, "per_seed": [ 2.4092166423797607, 2.4376542568206787, 2.0347108840942383, 1.5852525234222412, 1.881182074546814, 1.8407953977584839, 3.4355263710021973, 2.0861151218414307 ], "n": 8, "best_cfg": { "lr": 0.03 }, "sweep": [ { "cfg": { "lr": 0.003 }, "mean": 7.919645547866821, "std": 2.5149207407136784, "per_seed": [ 10.004054069519043, 5.846197605133057, 7.366078853607178, 4.32836389541626, 7.836812496185303, 5.4305877685546875, 11.452704429626465, 11.092365264892578 ], "n": 8 }, { "cfg": { "lr": 0.01 }, "mean": 3.1929454803466797, "std": 0.8334051410010931, "per_seed": [ 3.184743642807007, 3.0707268714904785, 3.08829665184021, 2.054521322250366, 2.8313469886779785, 2.428166151046753, 4.886856555938721, 3.998905658721924 ], "n": 8 }, { "cfg": { "lr": 0.03 }, "mean": 2.2138066589832306, "std": 0.5331661449712295, "per_seed": [ 2.4092166423797607, 2.4376542568206787, 2.0347108840942383, 1.5852525234222412, 1.881182074546814, 1.8407953977584839, 3.4355263710021973, 2.0861151218414307 ], "n": 8 } ] }, "comparison": { "delta_mean": -0.2009124755859375, "idea_wins": 6, "n_pairs": 8, "per_seed_diffs": [ 0.06673002243041992, -0.056865692138671875, -0.7018630504608154, -0.831536054611206, -0.33622872829437256, -0.217401385307312, -0.1770470142364502, -0.7071819305419922 ], "p_value": 0.41835, "mde": 0.5395513694218982, "mde_rel_pct": 22.34427025891624, "verdict": "no significant win", "system_worked": false }, "custom_track": { "name": "stochastic_intervention_scm", "file": "custom_stochastic_intervention_track.py", "domain": "causal-world-model" }, "mechanism_signature": { "baseline_predicted_selected_mean": 1.6279271692037582, "idea_predicted_selected_mean": 1.8218559473752975, "observed_selected_mean": 1.7572368532419205, "baseline_abs_error": 0.12930968403816223, "idea_abs_error": 0.06461909413337708, "confirmed": true } }, "how_to_run": "/home/maxwelhelp/main/bin/python3 official_bench_run.py", "files": [ "custom_stochastic_intervention_track.py", "official_bench_run.py", "bench_report.json" ], "limitations": "The custom registered track is synthetic rather than a real-world dataset. The official runner uses a custom joint-NLL loop for the idea because the method changes the loss, while the baseline uses bench.train_model; the shared outcome architecture and training budget are matched, but the joint model has additional q and mediator heads required by the idea. Only one registered custom track and one base architecture were tested.", "system_verdict": "partial", "practical_verdict": "inconclusive", "mechanism_ok": 1, "system_judged": true }