feat(eval): 多 provider/车道扫描 bench — 用数据给每个 agent 选最优执行路径

evals/bench_providers.py:同一 golden 任务自动扫一组 (provider,model,fc) 组合,
default_judge 打分出对比表(COMBOS 可配)。回答"该用 claude-code文本/qwen文本/qwen-FC"。

首轮 literature-mapper 实测(反直觉、印证"必须实验"):
- claude-code·文本 = 0.10/339s/反复卡死 → 最差(纠正"claude-code 更强"的误判:
  非 workspace 的 claude-code agent 走 react-over-text、不走 native,是最脆路径)。
- qwen-plus·文本 = 0.80 这轮最高。
- qwen-plus·FC = 0.40 翻车(没落盘):literature-mapper 无 enforceLoop,FC 可能
  "文本答了不写文件就收尾"——暴露 FC 真实风险:会写文件的 agent 设 FC 前需配 enforceLoop 计数。
结论:FC 非稳赢(LLM 方差大,需多轮均值);但 claude-code·文本对研究 agent 确为最差,自建该搬离。

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

evals/bench_providers.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python3
+"""多 provider / 车道扫描 benchmark — 用数据决定每个 agent 该用哪条执行路径。
+
+同一 golden 任务,自动跑一组 (provider, model, nativeToolCalls) 组合,收产物 →
+default_judge 打分 → 出对比表。回答"这个 agent 用 claude-code-文本 / qwen-文本 /
+qwen-FC 哪个质量高"——而非拍脑袋。
+
+纠正一个常见误判:非 workspace 的 claude-code agent **不走 native 车道**(那只给
+workspace.assistant),而是 react-over-text + claude-code 当纯文本 provider —— 最脆的路。
+所以"换 qwen-FC 是降级"未必成立,要扫描才知道。
+
+用法:
+  python3 evals/bench_providers.py [golden/literature-mapper.yaml]
+  COMBOS="claude-code:sonnet:0,dashscope:qwen-plus:0,dashscope:qwen-plus:1" python3 evals/bench_providers.py
+要 ~/.agentpaas/.env(各 provider key)+ 本地 ollama(judge)。真调 LLM、花钱。
+"""
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import sys
+import tempfile
+import time
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "agentpaas", "src"))
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "lambdagent", "src"))
+
+_envf = os.path.expanduser("~/.agentpaas/.env")
+if os.path.exists(_envf):
+    for line in open(_envf):
+        if "=" in line and not line.strip().startswith("#"):
+            k, v = line.strip().split("=", 1)
+            os.environ.setdefault(k.strip(), v.strip().strip('"').strip("'"))
+
+import sqlite3                                              # noqa: E402
+import yaml                                                # noqa: E402
+from lambdagent.fromconfig import from_config              # noqa: E402
+from lambdagent.core import Context                        # noqa: E402
+from lambdagent.builtin_tools.shell_tools import _set_cwd  # noqa: E402
+from lambdagent.builtin_tools._sandbox import set_sandbox_root  # noqa: E402
+from agentpaas.engine.agent_eval import EvalTask, run_eval_task  # noqa: E402
+from agentpaas.engine.pipeline import default_judge        # noqa: E402
+
+DB = os.path.expanduser("~/.agentpaas/data/agentpaas.db")
+MAXSTEPS = int(os.environ.get("MAXSTEPS", "14"))
+
+# 组合:provider:model:fc(1/0)。默认 claude-code文本 vs qwen文本 vs qwen-FC。
+_DEFAULT = "claude-code:sonnet:0,dashscope:qwen-plus:0,dashscope:qwen-plus:1"
+COMBOS = [tuple(c.split(":")) for c in os.environ.get("COMBOS", _DEFAULT).split(",")]
+
+
+def _base_cfg(template: str) -> dict:
+    con = sqlite3.connect(DB); con.row_factory = sqlite3.Row
+    a = con.execute("SELECT id,current_version FROM agents WHERE agent_template=? "
+                    "AND status='active' LIMIT 1", (template,)).fetchone()
+    if not a:
+        raise RuntimeError(f"没有 {template} 的已安装实例")
+    cfg = json.loads(con.execute("SELECT config FROM agent_versions WHERE agent_id=? AND version=?",
+                                 (a["id"], a["current_version"])).fetchone()["config"])
+    cfg.setdefault("react", {})["maxSteps"] = MAXSTEPS
+    cfg["react"]["toolTimeout"] = 90
+    return cfg
+
+
+def _run_combo(base: dict, provider: str, model: str, fc: bool, task: EvalTask) -> dict:
+    d = tempfile.mkdtemp(prefix="bench_"); _set_cwd(d); set_sandbox_root(d)
+    try:
+        c = json.loads(json.dumps(base))
+        c["model"] = {"provider": provider, "name": model, "temperature": 0.0, "maxTokens": 4096}
+        c["react"]["nativeToolCalls"] = fc
+        with tempfile.NamedTemporaryFile("w", suffix=".yml", delete=False, encoding="utf-8") as f:
+            yaml.dump(c, f, allow_unicode=True); p = f.name
+        term = from_config(p); os.unlink(p)
+        ctx = Context(workspace_path=d, run_id="bench")
+        t0 = time.time(); out = term.apply(task.input, ctx); dt = time.time() - t0
+        files = {fn: open(os.path.join(d, fn), encoding="utf-8", errors="ignore").read()
+                 for fn in os.listdir(d) if os.path.isfile(os.path.join(d, fn))}
+        return {"output": str(out), "steps": len(getattr(ctx, "trace", []) or []),
+                "cost_usd": 0.0, "workspace_path": d, "_files": files, "_dt": dt}
+    finally:
+        set_sandbox_root(None); shutil.rmtree(d, ignore_errors=True)
+
+
+def main():
+    paths = sys.argv[1:] or [os.path.join(os.path.dirname(__file__), "golden", "literature-mapper.yaml")]
+    print(f"provider/车道扫描 | maxSteps={MAXSTEPS}")
+    print(f"组合: {['/'.join(c) for c in COMBOS]}\n")
+    report = {}
+    for p in paths:
+        spec = yaml.safe_load(open(p, encoding="utf-8"))
+        base = _base_cfg(spec["agent_template"])
+        for t in spec.get("tasks", []):
+            task = EvalTask(id=t["id"], input=t.get("input", ""),
+                            must_produce=t.get("must_produce", []),
+                            rubric=t.get("rubric", ""), threshold=float(t.get("threshold", 0.6)))
+            print(f"=== {spec['agent_template']} / {task.id} ===")
+            rows = {}
+            for provider, model, fcs in COMBOS:
+                fc = fcs in ("1", "true", "True")
+                label = f"{provider}/{model}{'·FC' if fc else '·文本'}"
+                holder = {}
+
+                def wrapped(tk, _p=provider, _m=model, _fc=fc, _h=holder):
+                    r = _run_combo(base, _p, _m, _fc, tk); _h.update(r); return r
+                try:
+                    r = run_eval_task(task, run_fn=wrapped,
+                                      collect_artifacts_fn=lambda ws, _h=holder: _h.get("_files", {}),
+                                      judge_fn=default_judge)
+                    rows[label] = {"files_ok": r.files_ok, "judge": round(r.judge_score, 2),
+                                   "passed": r.passed, "dt": round(holder.get("_dt", 0)),
+                                   "err": r.error[:50]}
+                    print(f"  {label:28} files_ok={r.files_ok} judge={r.judge_score:.2f} "
+                          f"passed={r.passed} {round(holder.get('_dt',0))}s"
+                          + (f" err:{r.error[:40]}" if r.error else ""))
+                except Exception as e:
+                    rows[label] = {"err": str(e)[:80]}
+                    print(f"  {label:28} ✗ {str(e)[:60]}")
+            report[task.id] = rows
+            best = max((v for v in rows.values() if "judge" in v),
+                       key=lambda v: v["judge"], default=None)
+            if best:
+                win = [k for k, v in rows.items() if v.get("judge") == best["judge"]]
+                print(f"  → 最高分: {best['judge']:.2f}  ({', '.join(win)})\n")
+    out = os.path.join(os.path.dirname(__file__), "bench_providers_report.json")
+    json.dump(report, open(out, "w", encoding="utf-8"), ensure_ascii=False, indent=2)
+    print(f"报告: {out}")
+
+
+if __name__ == "__main__":
+    main()

evals/bench_providers_report.json

@@ -0,0 +1,25 @@
+{
+  "sla-lit-map": {
+    "claude-code/sonnet·文本": {
+      "files_ok": true,
+      "judge": 0.1,
+      "passed": false,
+      "dt": 339,
+      "err": ""
+    },
+    "dashscope/qwen-plus·文本": {
+      "files_ok": true,
+      "judge": 0.8,
+      "passed": true,
+      "dt": 90,
+      "err": ""
+    },
+    "dashscope/qwen-plus·FC": {
+      "files_ok": false,
+      "judge": 0.4,
+      "passed": false,
+      "dt": 23,
+      "err": ""
+    }
+  }
+}