import { mkdir, rename } from "node:fs/promises" import path from "node:path" import { parseArgs } from "node:util" import { asRecord, digest, getString, loadAgentPaaSConfig, canonicalJSON } from "../.opencode/lib/agentpaas" import run from "../.opencode/tool/agentpaas_run" import { agents, check, upload, task, download } from "../.opencode/tool/agentpaas" import { evaluateAcceptance, type AcceptanceCase } from "./lib/agentpaas-acceptance" const suite = path.resolve(import.meta.dir, "../evals/agentpaas-acceptance") const options = parseArgs({ options: { run: { type: "boolean" }, resume: { type: "boolean" }, catalog: { type: "boolean" }, case: { type: "string" }, "agent-id": { type: "string" }, output: { type: "string" }, "model-label": { type: "string" }, "wait-seconds": { type: "string", default: "120" }, "probe-model": { type: "boolean" }, }, }).values const cases = (await Bun.file(path.join(suite, "cases.json")).json()) as AcceptanceCase[] const context = { sessionID: crypto.randomUUID(), messageID: "acceptance", agent: "paas", directory: suite, worktree: suite, abort: new AbortController().signal, metadata() {}, async ask(request: { permission: string }) { if (request.permission !== "agentpaas_upload" && request.permission !== "agentpaas_download") throw new Error("验收脚本不自动批准智能体工具操作,请从 OpenCode 查看并明确决定。") }, } function structured(result: Awaited>) { if (typeof result === "string" || !result.metadata) throw new Error("工具未返回结构化状态") return result } if (options.catalog) { console.log(structured(await agents.execute({}, context)).output) } else if (!options.run && !options.resume) { console.log( JSON.stringify( { mode: "plan_only", cases, instruction: "使用 --catalog 选择实际智能体;--run --case data --agent-id ag_... --model-label 实际模型版本 --output 空目录 启动一个用例。--resume --output 原目录 只观察同一任务。", }, null, 2, ), ) } else { if (options.run && options.resume) throw new Error("--run 与 --resume 不能同时使用") if (!options.output) throw new Error("必须指定 --output") const wait = Number(options["wait-seconds"]) if (!Number.isInteger(wait) || wait < 0 || wait > 900) throw new Error("wait-seconds 必须为 0–900;到期只停止观察,不取消服务端任务") const config = await loadAgentPaaSConfig() const scope = digest(canonicalJSON([config.url, config.apiKey])) const directory = path.resolve(options.output) const receiptPath = path.join(directory, "receipt.json") const previous = options.resume ? asRecord(await Bun.file(receiptPath).json()) : undefined const sample = cases.find((item) => item.id === (previous?.case_id || options.case)) if (!sample) throw new Error("请选择 data、qa、software 或 research 用例") if (previous && previous.scope !== scope) throw new Error("当前服务或账号与原任务不同;停止恢复") if (!previous && (!options["agent-id"] || !options["model-label"])) throw new Error("首次运行必须指定 --agent-id 和 --model-label;标签需与部署配置核对") if (!previous) await mkdir(directory, { recursive: false, mode: 0o700 }) const receipt: Record = previous ?? { case_id: sample.id, agent_id: options["agent-id"], model_label: options["model-label"], model_label_source: "operator_declared", execution_mode: sample.mode, scope, idempotency_key: `acceptance-${crypto.randomUUID()}`, session_id: context.sessionID, started_at: new Date().toISOString(), phase: "prepared", } context.sessionID = getString(receipt.session_id) const save = async () => { await Bun.write(`${receiptPath}.tmp`, JSON.stringify(receipt, null, 2) + "\n", { mode: 0o600 }) await rename(`${receiptPath}.tmp`, receiptPath) } await save() try { const agentID = getString(receipt.agent_id) if (!previous) { const probe = structured( await check.execute({ agent_id: agentID, check_model: options["probe-model"] ?? false }, context), ) receipt.preflight = probe.metadata?.preflight await save() if (asRecord(receipt.preflight)?.blocked) throw new Error(probe.output) const files = structured( await upload.execute({ paths: sample.files.map((file) => path.join(suite, file)) }, context), ) receipt.input_file_ids = (files.metadata?.files as Array<{ file_id: string }>).map((file) => file.file_id) receipt.phase = "submitting" await save() const accepted = structured( await run.execute( { agent_id: agentID, input: sample.prompt, input_file_ids: receipt.input_file_ids as string[], idempotency_key: getString(receipt.idempotency_key), background: true, conversation: "new", mode: sample.mode, }, context, ), ) receipt.run_id = accepted.metadata?.run_id receipt.phase = "observing" await save() } if (!receipt.run_id && previous && receipt.phase === "submitting") { const recovered = structured( await task.execute( { agent_id: agentID, action: "recover", idempotency_key: getString(receipt.idempotency_key) }, context, ), ) if (recovered.metadata?.run_id) { receipt.run_id = recovered.metadata.run_id receipt.phase = "observing" await save() } } const runID = getString(receipt.run_id) if (!runID) throw new Error("尚无受理编号;请先用 /paas-tasks 或 history 核查提交结果,禁止重新启动用例") const deadline = Date.now() + wait * 1000 let result = structured(await task.execute({ agent_id: agentID, run_id: runID, action: "status" }, context)) while (["running", "queued", "resuming"].includes(getString(result.metadata?.status)) && Date.now() < deadline) { result = structured( await task.execute( { agent_id: agentID, run_id: runID, action: "watch", wait_seconds: Math.min(30, Math.max(1, Math.ceil((deadline - Date.now()) / 1000))), }, context, ), ) if (Array.isArray(result.metadata?.warnings) && result.metadata.warnings.length) break } const metadata = result.metadata! const artifacts = (Array.isArray(metadata.artifacts) ? metadata.artifacts : []).flatMap((item) => { const row = asRecord(item) return row && getString(row.artifact_id) ? [row] : [] }) const downloads = asRecord(receipt.downloads) ?? {} receipt.downloads = downloads for (const item of artifacts.filter((item) => sample.required_artifacts.includes(getString(item.name)))) { const id = getString(item.artifact_id) if (typeof downloads[id] === "string" && (await Bun.file(downloads[id]).exists())) continue const fetched = structured( await download.execute({ agent_id: agentID, run_id: runID, artifact_id: id }, { ...context, directory }), ) downloads[id] = fetched.metadata?.local_path await save() } const target = artifacts.filter((item) => item.name === sample.artifact) const targetPath = target.length === 1 ? getString(downloads[getString(target[0].artifact_id)]) : "" const status = getString(metadata.status) const terminal = ["completed", "partial", "failed", "cancelled", "needs_clarification"].includes(status) const content = targetPath ? await Bun.file(targetPath).text() : undefined const contents: Record = {} for (const name of sample.required_artifacts) { const matches = artifacts.filter((item) => item.name === name) const local = matches.length === 1 ? getString(downloads[getString(matches[0].artifact_id)]) : "" if (local) contents[name] = await Bun.file(local).text() } const inputs = Object.fromEntries( await Promise.all(sample.files.map(async (name) => [name, await Bun.file(path.join(suite, name)).text()])), ) const evaluation = terminal ? evaluateAcceptance( sample, artifacts.map((item) => getString(item.name)), content, contents, inputs, ) : null const summary = { case_id: sample.id, agent_id: agentID, run_id: runID, model_label: receipt.model_label, execution_mode: receipt.execution_mode, usage_complete: asRecord(metadata.usage)?.usage_complete === true, cost_complete: asRecord(metadata.usage)?.cost_complete === true, status, observation_complete: terminal, wall_seconds_since_submission: (Date.now() - Date.parse(getString(receipt.started_at))) / 1000, total_tokens: metadata.total_tokens ?? asRecord(metadata.usage)?.total_tokens ?? null, cost_usd_reported: metadata.cost_usd ?? asRecord(metadata.usage)?.cost_usd ?? null, usage_source: "execution_service; null means unreported; zero is not a verified free-cost claim", automatic_checks: evaluation, correction_rounds: 0, quality_score: null, note: terminal ? "自动检查只验证文件与预定值;人工项未评审,不计算综合质量总分。" : "任务未结束或等待操作。到期不取消、不重跑;处理审批后用 --resume 继续观察。", } await Bun.write(path.join(directory, "result.json"), JSON.stringify(summary, null, 2) + "\n", { mode: 0o600 }) await Bun.write(path.join(directory, "result.md"), result.output + "\n", { mode: 0o600 }) receipt.phase = terminal ? "finished" : "pending" await save() console.log(JSON.stringify(summary, null, 2)) } catch (error) { receipt.error = String(error).replaceAll(config.apiKey, "[redacted]") await save() console.error(receipt.error) process.exitCode = 1 } }