feat(eval): bench 加多轮取均值(RUNS) — 消 LLM 方差,信号扎实

RUNS 环境变量:每个 provider/车道组合重复跑 N 次,_agg 聚合 judge 均值/极值/落盘率/
通过率。批量多 agent 已有(传多个 golden yaml)。

RUNS=2 验证(literature-mapper,qwen-plus):FC 均 0.88 [0.85~0.90] 100%落盘,文本均
0.82 [0.80~0.85]——之前单轮 FC 翻车的 0.40 被证明是离群点,均值后 FC 反超且快近一倍。
结论:多轮均值能把 LLM 方差抹平,把"FC vs 文本"从拍脑袋变成可信数字。N>=5 更稳。

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

evals/bench_providers.py

@@ -44,6 +44,21 @@ from agentpaas.engine.pipeline import default_judge        # noqa: E402
 
 DB = os.path.expanduser("~/.agentpaas/data/agentpaas.db")
 MAXSTEPS = int(os.environ.get("MAXSTEPS", "14"))
+RUNS = int(os.environ.get("RUNS", "1"))   # 每个组合重复跑几次取均值(消 LLM 方差)
+
+
+def _agg(results: list) -> dict:
+    """对同一组合的 N 次 EvalResult 聚合:均值 + 极值 + 达标率。"""
+    n = len(results) or 1
+    js = [r.judge_score for r in results]
+    return {
+        "runs": len(results),
+        "judge_avg": round(sum(js) / n, 2),
+        "judge_min": round(min(js), 2) if js else 0.0,
+        "judge_max": round(max(js), 2) if js else 0.0,
+        "files_ok_rate": round(sum(1 for r in results if r.files_ok) / n, 2),
+        "pass_rate": round(sum(1 for r in results if r.passed) / n, 2),
+    }
 
 # 组合:provider:model:fc(1/0)。默认 claude-code文本 vs qwen文本 vs qwen-FC。
 _DEFAULT = "claude-code:sonnet:0,dashscope:qwen-plus:0,dashscope:qwen-plus:1"
@@ -99,29 +114,33 @@ def main():
             for provider, model, fcs in COMBOS:
                 fc = fcs in ("1", "true", "True")
                 label = f"{provider}/{model}{'·FC' if fc else '·文本'}"
-                holder = {}
-
-                def wrapped(tk, _p=provider, _m=model, _fc=fc, _h=holder):
-                    r = _run_combo(base, _p, _m, _fc, tk); _h.update(r); return r
-                try:
-                    r = run_eval_task(task, run_fn=wrapped,
-                                      collect_artifacts_fn=lambda ws, _h=holder: _h.get("_files", {}),
-                                      judge_fn=default_judge)
-                    rows[label] = {"files_ok": r.files_ok, "judge": round(r.judge_score, 2),
-                                   "passed": r.passed, "dt": round(holder.get("_dt", 0)),
-                                   "err": r.error[:50]}
-                    print(f"  {label:28} files_ok={r.files_ok} judge={r.judge_score:.2f} "
-                          f"passed={r.passed} {round(holder.get('_dt',0))}s"
-                          + (f" err:{r.error[:40]}" if r.error else ""))
-                except Exception as e:
-                    rows[label] = {"err": str(e)[:80]}
-                    print(f"  {label:28} ✗ {str(e)[:60]}")
+                run_results = []
+                for run_i in range(RUNS):
+                    holder = {}
+
+                    def wrapped(tk, _p=provider, _m=model, _fc=fc, _h=holder):
+                        r = _run_combo(base, _p, _m, _fc, tk); _h.update(r); return r
+                    try:
+                        rr = run_eval_task(task, run_fn=wrapped,
+                                           collect_artifacts_fn=lambda ws, _h=holder: _h.get("_files", {}),
+                                           judge_fn=default_judge)
+                        run_results.append(rr)
+                        print(f"  {label:26} run{run_i+1}/{RUNS} judge={rr.judge_score:.2f} "
+                              f"files_ok={rr.files_ok} {round(holder.get('_dt',0))}s"
+                              + (f" err:{rr.error[:30]}" if rr.error else ""))
+                    except Exception as e:
+                        print(f"  {label:26} run{run_i+1}/{RUNS} ✗ {str(e)[:50]}")
+                if run_results:
+                    rows[label] = _agg(run_results)
             report[task.id] = rows
-            best = max((v for v in rows.values() if "judge" in v),
-                       key=lambda v: v["judge"], default=None)
-            if best:
-                win = [k for k, v in rows.items() if v.get("judge") == best["judge"]]
-                print(f"  → 最高分: {best['judge']:.2f}  ({', '.join(win)})\n")
+            print(f"  --- {task.id} 聚合({RUNS} 轮均值)---")
+            for lbl, a in sorted(rows.items(), key=lambda kv: -kv[1]["judge_avg"]):
+                print(f"    {lbl:26} judge均={a['judge_avg']:.2f} "
+                      f"[{a['judge_min']:.2f}~{a['judge_max']:.2f}] "
+                      f"落盘率={a['files_ok_rate']:.0%} 通过率={a['pass_rate']:.0%}")
+            if rows:
+                best = max(rows.items(), key=lambda kv: kv[1]["judge_avg"])
+                print(f"  → 最优: {best[0]}  (judge均 {best[1]['judge_avg']:.2f})\n")
     out = os.path.join(os.path.dirname(__file__), "bench_providers_report.json")
     json.dump(report, open(out, "w", encoding="utf-8"), ensure_ascii=False, indent=2)
     print(f"报告: {out}")

evals/bench_providers_report.json

@@ -1,25 +1,20 @@
 {
   "sla-lit-map": {
-    "claude-code/sonnet·文本": {
-      "files_ok": true,
-      "judge": 0.1,
-      "passed": false,
-      "dt": 339,
-      "err": ""
-    },
     "dashscope/qwen-plus·文本": {
-      "files_ok": true,
-      "judge": 0.8,
-      "passed": true,
-      "dt": 90,
-      "err": ""
+      "runs": 2,
+      "judge_avg": 0.82,
+      "judge_min": 0.8,
+      "judge_max": 0.85,
+      "files_ok_rate": 1.0,
+      "pass_rate": 1.0
     },
     "dashscope/qwen-plus·FC": {
-      "files_ok": false,
-      "judge": 0.4,
-      "passed": false,
-      "dt": 23,
-      "err": ""
+      "runs": 2,
+      "judge_avg": 0.88,
+      "judge_min": 0.85,
+      "judge_max": 0.9,
+      "files_ok_rate": 1.0,
+      "pass_rate": 1.0
     }
   }
 }