recovery.ts 5.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147
  1. import { mkdir, readdir, rename, rm } from "node:fs/promises"
  2. import path from "node:path"
  3. import { setTimeout } from "node:timers/promises"
  4. import {
  5. agentPaaSConversationDir,
  6. asRecord,
  7. canonicalJSON,
  8. digest,
  9. getString,
  10. requestJSON,
  11. trackRun,
  12. type Connection,
  13. } from "./agentpaas"
  14. function directory(config: Connection) {
  15. return path.join(agentPaaSConversationDir(), "recovery", digest(canonicalJSON([config.url, config.apiKey])))
  16. }
  17. function filename(config: Connection, kind: string, agentID: string, id: string) {
  18. return path.join(directory(config), `${kind}-${digest(canonicalJSON([agentID, id]))}.json`)
  19. }
  20. async function write(file: string, value: unknown) {
  21. await mkdir(path.dirname(file), { recursive: true, mode: 0o700 })
  22. const temporary = `${file}.${crypto.randomUUID()}.tmp`
  23. try {
  24. await Bun.write(temporary, JSON.stringify(value), { mode: 0o600 })
  25. await rename(temporary, file)
  26. } finally {
  27. await rm(temporary, { force: true })
  28. }
  29. }
  30. export async function pendingRequest(config: Connection, agentID: string, key: string) {
  31. await write(filename(config, "request", agentID, key), { agent_id: agentID, idempotency_key: key })
  32. }
  33. export async function clearPendingRequest(config: Connection, agentID: string, key: string) {
  34. await rm(filename(config, "request", agentID, key), { force: true })
  35. }
  36. export async function recoverRequest(config: Connection, agentID: string, key: string, signal: AbortSignal) {
  37. const receipt = await requestJSON(
  38. config,
  39. `/agents/${encodeURIComponent(agentID)}/requests/${encodeURIComponent(key)}`,
  40. signal,
  41. )
  42. const run = asRecord(receipt.run)
  43. if (receipt.found !== true || !getString(run?.run_id)) return undefined
  44. await trackRun(config, {
  45. agent_id: agentID,
  46. agent_name: agentID,
  47. run_id: getString(run!.run_id),
  48. thread_id: getString(run!.session_id) || getString(run!.thread_id),
  49. })
  50. await clearPendingRequest(config, agentID, key)
  51. return run
  52. }
  53. export async function recoverPendingRequests(config: Connection, signal: AbortSignal) {
  54. const files = await readdir(directory(config)).catch((error: NodeJS.ErrnoException) => {
  55. if (error.code === "ENOENT") return []
  56. throw error
  57. })
  58. const warnings: string[] = []
  59. // Bound each refresh; keep unresolved receipts for a later read-only retry.
  60. for (const file of files.filter((file) => /^request-[a-f0-9]{64}\.json$/.test(file)).slice(0, 20)) {
  61. const row = asRecord(
  62. await Bun.file(path.join(directory(config), file))
  63. .json()
  64. .catch(() => undefined),
  65. )
  66. if (!row) {
  67. warnings.push("提交凭据损坏,请查询服务端历史。")
  68. continue
  69. }
  70. const agentID = getString(row.agent_id),
  71. key = getString(row.idempotency_key)
  72. try {
  73. if (!(await recoverRequest(config, agentID, key, signal)))
  74. warnings.push(`${agentID} 的提交尚未查到,仍保留凭据;不要换键重跑。idempotency_key=${key}`)
  75. } catch (error) {
  76. if (signal.aborted) throw error
  77. warnings.push(`提交结果暂不可核查:${agentID};${String(error)}`)
  78. }
  79. }
  80. return warnings
  81. }
  82. export async function observation(config: Connection, agentID: string, runID: string) {
  83. const file = Bun.file(filename(config, "run", agentID, runID))
  84. return (await file.exists()) ? (asRecord(await file.json()) ?? {}) : {}
  85. }
  86. export async function saveObservation(
  87. config: Connection,
  88. agentID: string,
  89. runID: string,
  90. update: Record<string, unknown>,
  91. ) {
  92. await write(filename(config, "run", agentID, runID), { ...(await observation(config, agentID, runID)), ...update })
  93. }
  94. export async function cancelAndObserve(config: Connection, agentID: string, runID: string) {
  95. const base = `/agents/${encodeURIComponent(agentID)}/runs/${encodeURIComponent(runID)}`
  96. let persistenceWarning = ""
  97. const persist = async (state: string) => {
  98. await saveObservation(config, agentID, runID, { cancellation_status: state }).catch(() => {
  99. persistenceWarning = "本地取消记录未保存;请使用 run_id 查询服务端状态。"
  100. })
  101. }
  102. await persist("requested")
  103. const signal = AbortSignal.timeout(6000)
  104. let error = ""
  105. await requestJSON(config, `${base}/cancel`, signal, {
  106. method: "POST",
  107. headers: { "Content-Type": "application/json" },
  108. body: "{}",
  109. }).catch((cause) => {
  110. error = String(cause)
  111. })
  112. let snapshot: Record<string, unknown> = {}
  113. for (let attempt = 0; attempt < 3 && !signal.aborted; attempt++) {
  114. snapshot = await requestJSON(config, base, signal).catch(() => snapshot)
  115. if (["cancelled", "completed", "partial", "failed"].includes(getString(snapshot.status))) break
  116. await setTimeout(300 * (attempt + 1), undefined, { signal }).catch(() => undefined)
  117. }
  118. const state =
  119. snapshot.status === "cancelled"
  120. ? "confirmed"
  121. : ["completed", "partial", "failed"].includes(getString(snapshot.status))
  122. ? "already_terminal"
  123. : error
  124. ? "unconfirmed"
  125. : "requested"
  126. await persist(state)
  127. return {
  128. ...snapshot,
  129. run_id: runID,
  130. status: snapshot.status || "unknown",
  131. cancellation_status: state,
  132. ...(persistenceWarning ? { persistence_warning: persistenceWarning } : {}),
  133. }
  134. }
  135. export function cancellationStatus(saved: Record<string, unknown>, status: unknown) {
  136. if (!saved.cancellation_status) return undefined
  137. return status === "cancelled"
  138. ? "confirmed"
  139. : ["completed", "partial", "failed"].includes(getString(status))
  140. ? "already_terminal"
  141. : saved.cancellation_status
  142. }