opencode/script/lib/agentpaas-acceptance.ts

import { asRecord, canonicalJSON } from "../../.opencode/lib/agentpaas"

export type AcceptanceCase = {
  id: string
  mode: "chat" | "iterate"
  capability: string
  files: string[]
  prompt: string
  artifact: string
  expected: Record<string, unknown>
  required_artifacts: string[]
  review: string[]
}

// These are observable contract checks, not an LLM judgment of scientific quality.
export function evaluateAcceptance(
  sample: AcceptanceCase,
  artifacts: string[],
  content?: string,
  files: Record<string, string> = {},
  inputs: Record<string, string> = {},
) {
  const checks = sample.required_artifacts.map((name) => ({
    name: `artifact:${name}`,
    passed: artifacts.includes(name),
  }))
  const parsed =
    content === undefined
      ? undefined
      : (() => {
          try {
            return asRecord(JSON.parse(content))
          } catch {
            return undefined
          }
        })()
  checks.push({ name: "valid_json_object", passed: parsed !== undefined })
  for (const [name, expected] of Object.entries(sample.expected))
    checks.push({
      name: `value:${name}`,
      passed: parsed !== undefined && canonicalJSON(parsed[name]) === canonicalJSON(expected),
    })
  // Compare downloaded outputs to supplied inputs, never trust the model's own scores.
  if (sample.id === "data") {
    const rows = csv(inputs["measurements.csv"])
    const actual = csv(files["cleaned.csv"])
    const unique = rows?.filter((row, index, all) => all.findIndex((other) => other.id === row.id) === index)
    const clean = unique?.filter((row) => row.value.trim() !== "")
    checks.push({
      name: "independent:cleaned_rows",
      passed: !!clean && !!actual && canonicalJSON(actual) === canonicalJSON(clean),
    })
    const values = clean?.map((row) => Number(row.value))
    checks.push({
      name: "independent:recomputed_statistics",
      passed:
        !!rows &&
        !!unique &&
        !!clean &&
        !!values?.length &&
        values.every(Number.isFinite) &&
        !!parsed &&
        parsed.valid_rows === clean.length &&
        parsed.excluded_missing === unique.length - clean.length &&
        parsed.duplicates_removed === rows.length - unique.length &&
        parsed.overall_mean === values.reduce((a, b) => a + b, 0) / values.length &&
        ["A", "B"].every((group) => {
          const selected = clean.filter((row) => row.group === group).map((row) => Number(row.value))
          return (
            selected.length > 0 &&
            asRecord(parsed.group_means)?.[group] === selected.reduce((a, b) => a + b, 0) / selected.length
          )
        }),
    })
  }
  if (["qa", "research"].includes(sample.id)) {
    const source = inputs[sample.id === "qa" ? "policy.txt" : "evidence.md"] ?? ""
    const ids = sample.id === "qa" ? ["P1", "P2", "P3"] : ["E1", "E2", "E3"]
    const quotes = asRecord(parsed?.evidence_quotes)
    checks.push({
      name: "independent:source_quotes",
      passed:
        !!quotes &&
        !!source &&
        canonicalJSON(Object.keys(quotes).sort()) === canonicalJSON(ids) &&
        ids.every((id) => {
          const quote = quotes[id]
          const line = source.split(/\r?\n/).find((line) => line.startsWith(`[${id}]`) || line.startsWith(`- ${id}：`))
          return (
            typeof quote === "string" &&
            quote.trim().length >= 12 &&
            !!line?.includes(quote) &&
            !!files["report.md"]?.includes(quote)
          )
        }),
    })
    checks.push({
      name: "independent:evidence_ids",
      passed:
        Array.isArray(parsed?.evidence_ids) && canonicalJSON([...parsed.evidence_ids].sort()) === canonicalJSON(ids),
    })
  }
  return {
    checks,
    passed: checks.filter((check) => check.passed).length,
    total: checks.length,
    human_review: sample.review.map((criterion) => ({ criterion, result: "not_reviewed" })),
  }
}

// The fixed data fixture has three unquoted columns; reject ambiguous formats.
function csv(content?: string) {
  if (!content) return undefined
  const lines = content.trim().split(/\r?\n/)
  if (lines.shift()?.replace(/^\uFEFF/, "") !== "id,group,value") return undefined
  const rows = lines.map((line) => line.split(","))
  if (rows.some((row) => row.length !== 3)) return undefined
  return rows.map(([id, group, value]) => ({ id, group, value }))
}