# Стенд-проверка (stage-2) · промт оператора:

(универсальный)

Ответ агента:

{
  "worked": false,
  "confidence": 10,
  "verdict": "Built a structurally matched custom peer-prediction classification track and trained identical MLP systems with supervised baseline versus label-free BTS policy-gradient rewards. The trained-model signature confirmed the observed/predicted frequency log-ratio calculation, but BTS had substantially worse test error: 0.328750 versus 0.016875, with paired delta +0.311875 and permutation p=0.0081. Therefore the mechanism was numerically present but did not transfer into a benchmark win.",
  "metrics": {
    "baseline": "Best lr=0.003; 8-seed test error mean 0.016875, std 0.007474.",
    "idea": "Best lr=0.003; 8-seed test error mean 0.328750, std 0.184877; paired delta +0.311875, p=0.0081, 0/8 wins."
  },
  "bench_report": {
    "bench_version": 1,
    "track": "belief_sensitive_peer_prediction",
    "model": "mlp_tiny",
    "metric_direction": "lower is better",
    "n_seeds": 8,
    "baseline": {
      "best_cfg": {
        "lr": 0.003
      },
      "sweep": [
        {
          "cfg": {
            "lr": 0.001
          },
          "mean": 0.01625
        },
        {
          "cfg": {
            "lr": 0.003
          },
          "mean": 0.015
        },
        {
          "cfg": {
            "lr": 0.01
          },
          "mean": 0.02
        }
      ],
      "full": {
        "mean": 0.016875,
        "std": 0.007474,
        "per_seed": [
          0.01,
          0.02,
          0.025,
          0.005,
          0.015,
          0.025,
          0.01,
          0.025
        ],
        "n": 8
      }
    },
    "idea": {
      "mean": 0.32875,
      "std": 0.184877,
      "per_seed": [
        0.11,
        0.165,
        0.68,
        0.26,
        0.17,
        0.305,
        0.52,
        0.42
      ],
      "n": 8
    },
    "comparison": {
      "delta_mean": 0.311875,
      "idea_wins": 0,
      "n_pairs": 8,
      "p_value": 0.0081,
      "verdict": "idea worse (significant)",
      "system_worked": false
    },
    "mechanism_signature": {
      "group_size": 8,
      "n_model_completions": 1024,
      "predicted_mean_frequency": 0.513152,
      "observed_mean_frequency": 0.577148,
      "predicted_observed_correlation": 0.400833,
      "log_ratio_max_error": 0.0,
      "confirmed": true
    }
  },
  "how_to_run": "/home/maxwelhelp/main/bin/python3 run_bench.py",
  "files": [
    "bts_track.py",
    "run_bench.py",
    "bench_report.json"
  ],
  "limitations": "This was a small synthetic custom binary classification track rather than language-model GRPO on natural-language factual prompts. It did not test parsing failures, larger completion groups, pretrained models, preference/SFT baselines, or multi-turn pressure robustness.",
  "system_verdict": "partial",
  "practical_verdict": "harms",
  "mechanism_ok": 1,
  "system_judged": true
}