Стенд-проверка (stage-2) · промт оператора:

(универсальный)

Ответ агента:

{ "worked": false, "confidence": 10, "verdict": "Implemented a matched dense-vs-block-TT 3D Cartesian voxel operator benchmark with a local custom track, canonical train_model training, equal learning-rate sweeps, and 8 paired seeds. The trained TT operator used 512 parameters versus 16,384 dense operator parameters and reproduced its sequential map with relative error 8.13e-08, confirming the mechanism prediction. However, test MSE was significantly worse for Block-TT: delta_mean +1.1413 with permutation p=0.0081, so the idea did not win the benchmark.", "metrics": { "baseline": "Best lr=0.01; full 8-seed test MSE mean 0.0458705, std 0.0489606; dense operator 16,384 parameters.", "idea": "Best lr=0.01; full 8-seed test MSE mean 1.18719, std 1.00865; trained TT operator 512 parameters, 3.125% of dense operator storage; trained forward relative error 8.13e-08." }, "how_to_run": "/home/maxwelhelp/main/bin/python3 bench_block_tt.py", "files": [ "bench_block_tt.py", "voxel_operator_track.py", "bench_output.json" ], "limitations": "The custom benchmark is a small 2-channel 4x4x4 synthetic voxel operator task rather than a 64^3 PDE or Navier-Stokes workload. Throughput, peak activation memory, optimizer-time TT rounding, ranks 4/8/16, CP/tensor-ring comparison, and larger-scale neural-operator training were not tested. The TT model has substantially lower representational capacity than the dense baseline, so this demonstrates an accuracy-memory tradeoff rather than an equal-parameter comparison.", "bench_report": { "bench_version": 1, "track": "voxel_operator_3d", "model": "dense_vs_block_tt", "metric_direction": "lower is better", "n_seeds": 8, "baseline": { "best_cfg": { "lr": 0.01 }, "sweep": [ { "cfg": { "lr": 0.001 }, "mean": 21.742855668067932 }, { "cfg": { "lr": 0.003 }, "mean": 1.331242160871625 }, { "cfg": { "lr": 0.01 }, "mean": 0.0431183404289186 } ], "full": { "mean": 0.045870458823628724, "std": 0.048960569855847756, "per_seed": [ 0.08236338198184967, 0.020615510642528534, 0.009508499875664711, 0.059985969215631485, 0.012797527015209198, 0.1569584608078003, 0.014212244190275669, 0.010522076860070229 ], "n": 8 } }, "idea": { "mean": 1.1871879938989878, "std": 1.0086499807369265, "per_seed": [ 2.5437099933624268, 0.5915095210075378, 0.20690876245498657, 1.849648356437683, 0.36077338457107544, 2.847456932067871, 0.9292405247688293, 0.168256476521492 ], "n": 8, "best_cfg": { "lr": 0.01 }, "configs_tried": [ { "cfg": { "lr": 0.001 }, "mean": 52.323336124420166 }, { "cfg": { "lr": 0.003 }, "mean": 14.277131646871567 }, { "cfg": { "lr": 0.01 }, "mean": 1.1871879938989878 } ] }, "comparison": { "delta_mean": 1.141317535075359, "idea_wins": 0, "n_pairs": 8, "per_seed_diffs": [ 2.461346611380577, 0.5708940105793219, 0.19740026257932186, 1.7896623872220516, 0.34797585755586624, 2.690498471260071, 0.9150282805785537, 0.15773439966142178 ], "p_value": 0.0081, "mde": 0.8612897455089487, "mde_rel_pct": 1877.6567045483364, "verdict": "idea worse (significant)", "system_worked": false }, "custom_track": { "name": "voxel_operator_3d", "file": "voxel_operator_track.py", "domain": "3d_operator" }, "protocol_note": "8 paired seeds; custom 3-D Cartesian track; baseline and idea used the same learning-rate union and canonical train_model path.", "mechanism_signature": { "prediction": "A trained block-TT operator reduces operator storage while preserving its sequential forward map.", "predicted_operator_params": 16384, "observed_trained_tt_operator_params": 512, "observed_storage_ratio": 0.03125, "observed_trained_forward_relative_error": 8.130663076144629e-08, "confirmed": true } }, "system_verdict": "partial", "practical_verdict": "harms", "mechanism_ok": 1, "system_judged": true }