"""
agentpaas.engine.agent_eval — 第一方 agent 的"行为回归"评估框架。

评估建议②(docs/AGENT_QUALITY_ASSESSMENT.md):有 pipeline/gate 单测,缺第一方 agent
的行为回归集——"真实智能体质量"没被覆盖。本框架给每个 agent 配 golden 任务,跑完后量化:
落盘达标(must_produce)、LLM-judge 产物分、步数/成本。复用 pipeline 的 default_judge。

设计要点:run_eval_task 的"跑 agent / 收产物 / judge"全部**可注入** → 框架本身可用 mock
单测(进 CI);真实 LLM 跑由 evals/run_evals.py 手动/nightly 触发(要 API key、花钱)。
"""
from __future__ import annotations

from dataclasses import dataclass, field
from typing import Callable, Dict, List, Optional


@dataclass
class EvalTask:
    """一个 golden 评估任务。"""
    id: str
    agent_id: str = ""                       # 跑哪个 agent 实例
    agent_template: str = ""                 # 或按模板(runner 解析成实例)
    input: str = ""
    must_produce: List[str] = field(default_factory=list)  # 期望产物文件名(子串匹配)
    rubric: str = ""                         # LLM-judge 评分标准(空则跳过 judge)
    threshold: float = 0.6
    native_tool_calls: Optional[bool] = None  # 临时覆盖 react.nativeToolCalls(对比 FC vs 文本)


@dataclass
class EvalResult:
    task_id: str
    produced_files: List[str] = field(default_factory=list)
    files_ok: bool = True                    # must_produce 全部命中
    missing_files: List[str] = field(default_factory=list)
    judge_score: float = 0.0
    judge_passed: bool = True
    judge_reasons: List[str] = field(default_factory=list)
    steps: int = 0
    cost_usd: float = 0.0
    error: str = ""
    passed: bool = False                     # 总判定 = files_ok 且 judge_passed 且无 error


def run_eval_task(task: EvalTask, *,
                  run_fn: Callable[[EvalTask], dict],
                  collect_artifacts_fn: Callable[[str], Dict[str, str]],
                  judge_fn: Callable) -> EvalResult:
    """跑一个 golden 任务并打分。三个回调全可注入:
      run_fn(task) -> {"output", "steps", "cost_usd", "workspace_path"}
      collect_artifacts_fn(workspace_path) -> {filename: text}
      judge_fn(JudgeRequest) -> JudgeResult
    """
    try:
        run = run_fn(task)
    except Exception as e:  # 跑挂了 = 这条任务失败,不是框架崩
        return EvalResult(task_id=task.id, error=str(e)[:400], passed=False)

    files = collect_artifacts_fn(run.get("workspace_path", "")) or {}
    names = list(files.keys())
    missing = [m for m in task.must_produce
               if not any(m in n for n in names)]
    files_ok = not missing

    judge_score, judge_passed, reasons = 0.0, True, []
    if task.rubric:
        from agentpaas.engine.pipeline import JudgeRequest
        artifacts_text = "\n\n".join(f"## {n}\n{t[:4000]}" for n, t in files.items())
        if not artifacts_text:
            artifacts_text = str(run.get("output", ""))[:8000]
        jr = judge_fn(JudgeRequest(rubric=task.rubric, threshold=task.threshold,
                                   model="", artifacts_text=artifacts_text, stage_id=task.id))
        judge_score, judge_passed, reasons = jr.score, jr.passed, list(jr.reasons or [])

    err = str(run.get("output", "")) if _looks_like_error(run.get("output", "")) else ""
    passed = files_ok and judge_passed and not err
    return EvalResult(
        task_id=task.id, produced_files=names, files_ok=files_ok, missing_files=missing,
        judge_score=judge_score, judge_passed=judge_passed, judge_reasons=reasons,
        steps=int(run.get("steps", 0) or 0), cost_usd=float(run.get("cost_usd", 0.0) or 0.0),
        error=err, passed=passed,
    )


def _looks_like_error(output: str) -> bool:
    import re
    return bool(re.search(r"\[[A-Z0-9_-]*(ERROR|TIMEOUT)\]", str(output or "")))


def summarize(results: List[EvalResult]) -> dict:
    """汇总:通过率、落盘达标率、平均分/步数/成本。"""
    n = len(results) or 1
    return {
        "total": len(results),
        "passed": sum(1 for r in results if r.passed),
        "pass_rate": round(sum(1 for r in results if r.passed) / n, 3),
        "files_ok_rate": round(sum(1 for r in results if r.files_ok) / n, 3),
        "avg_judge_score": round(sum(r.judge_score for r in results) / n, 3),
        "avg_steps": round(sum(r.steps for r in results) / n, 1),
        "total_cost_usd": round(sum(r.cost_usd for r in results), 4),
        "failures": [r.task_id for r in results if not r.passed],
    }
