tool-stream.ts 6.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186
  1. import { Effect } from "effect"
  2. import { LLMError, LLMEvent, type ProviderMetadata, type ToolCall, type ToolInputDelta } from "../../schema"
  3. import { eventError, parseToolInput, type ToolAccumulator } from "../shared"
  4. type StreamKey = string | number
  5. /**
  6. * One pending streamed tool call. Providers emit the tool identity and JSON
  7. * argument text across separate chunks; `input` is the raw JSON string collected
  8. * so far, not the parsed object.
  9. */
  10. export interface PendingTool extends ToolAccumulator {
  11. readonly providerExecuted?: boolean
  12. readonly providerMetadata?: ProviderMetadata
  13. }
  14. /**
  15. * Sparse parser state keyed by the provider's stream-local tool identifier.
  16. *
  17. * This key is not the final tool-call id (`call_...`). It is the id/index the
  18. * provider uses while streaming a partial call: OpenAI Chat / Anthropic /
  19. * Bedrock use numeric content indexes, while OpenAI Responses uses string
  20. * `item_id`s. The generic keeps each protocol internally consistent.
  21. */
  22. export type State<K extends StreamKey> = Partial<Record<K, PendingTool>>
  23. /**
  24. * Result of adding argument text to one pending tool call. It returns both the
  25. * next `tools` state and the updated `tool` because parsers often need the
  26. * current id/name immediately. `event` is present only when new text arrived;
  27. * metadata-only deltas update identity without emitting `tool-input-delta`.
  28. */
  29. export interface AppendOutcome<K extends StreamKey> {
  30. readonly tools: State<K>
  31. readonly tool: PendingTool
  32. readonly event?: ToolInputDelta
  33. }
  34. /** Create empty accumulator state for one provider stream. */
  35. export const empty = <K extends StreamKey>(): State<K> => ({})
  36. const withTool = <K extends StreamKey>(tools: State<K>, key: K, tool: PendingTool): State<K> => {
  37. return { ...tools, [key]: tool }
  38. }
  39. const withoutTool = <K extends StreamKey>(tools: State<K>, key: K): State<K> => {
  40. const next = { ...tools }
  41. delete next[key]
  42. return next
  43. }
  44. const inputDelta = (tool: PendingTool, text: string): ToolInputDelta =>
  45. LLMEvent.toolInputDelta({
  46. id: tool.id,
  47. name: tool.name,
  48. text,
  49. })
  50. const toolCall = (route: string, tool: PendingTool, inputOverride?: string) =>
  51. parseToolInput(route, tool.name, inputOverride ?? tool.input).pipe(
  52. Effect.map(
  53. (input): ToolCall =>
  54. LLMEvent.toolCall({
  55. id: tool.id,
  56. name: tool.name,
  57. input,
  58. providerExecuted: tool.providerExecuted ? true : undefined,
  59. providerMetadata: tool.providerMetadata,
  60. }),
  61. ),
  62. )
  63. /** Store the updated tool and produce the optional public delta event. */
  64. const appendTool = <K extends StreamKey>(
  65. tools: State<K>,
  66. key: K,
  67. tool: PendingTool,
  68. text: string,
  69. ): AppendOutcome<K> => ({
  70. tools: withTool(tools, key, tool),
  71. tool,
  72. event: text.length === 0 ? undefined : inputDelta(tool, text),
  73. })
  74. export const isError = <K extends StreamKey>(result: AppendOutcome<K> | LLMError): result is LLMError =>
  75. result instanceof LLMError
  76. /**
  77. * Register a tool call whose start event arrived before any argument deltas.
  78. * Used by Anthropic `content_block_start`, Bedrock `contentBlockStart`, and
  79. * OpenAI Responses `response.output_item.added`.
  80. */
  81. export const start = <K extends StreamKey>(
  82. tools: State<K>,
  83. key: K,
  84. tool: Omit<PendingTool, "input"> & { readonly input?: string },
  85. ) => withTool(tools, key, { ...tool, input: tool.input ?? "" })
  86. /**
  87. * Append a streamed argument delta, starting the tool if this provider encodes
  88. * identity on the first delta instead of a separate start event. OpenAI Chat has
  89. * this shape: `tool_calls[].index` is the stream key, and `id` / `name` may only
  90. * appear on the first delta for that index.
  91. */
  92. export const appendOrStart = <K extends StreamKey>(
  93. route: string,
  94. tools: State<K>,
  95. key: K,
  96. delta: { readonly id?: string; readonly name?: string; readonly text: string },
  97. missingToolMessage: string,
  98. ): AppendOutcome<K> | LLMError => {
  99. const current = tools[key]
  100. const id = delta.id ?? current?.id
  101. const name = delta.name ?? current?.name
  102. if (!id || !name) return eventError(route, missingToolMessage)
  103. const tool = {
  104. id,
  105. name,
  106. input: `${current?.input ?? ""}${delta.text}`,
  107. providerExecuted: current?.providerExecuted,
  108. providerMetadata: current?.providerMetadata,
  109. }
  110. if (current && delta.text.length === 0 && current.id === id && current.name === name) return { tools, tool: current }
  111. return appendTool(tools, key, tool, delta.text)
  112. }
  113. /**
  114. * Append argument text to a tool that must already have been started. This keeps
  115. * protocols honest when their stream grammar promises a start event before any
  116. * argument delta.
  117. */
  118. export const appendExisting = <K extends StreamKey>(
  119. route: string,
  120. tools: State<K>,
  121. key: K,
  122. text: string,
  123. missingToolMessage: string,
  124. ): AppendOutcome<K> | LLMError => {
  125. const current = tools[key]
  126. if (!current) return eventError(route, missingToolMessage)
  127. if (text.length === 0) return { tools, tool: current }
  128. return appendTool(tools, key, { ...current, input: `${current.input}${text}` }, text)
  129. }
  130. /**
  131. * Finalize one pending tool call: parse the accumulated raw JSON, remove it
  132. * from state, and return the optional public `tool-call` event. Missing keys are
  133. * a no-op because some providers emit stop events for non-tool content blocks.
  134. */
  135. export const finish = <K extends StreamKey>(route: string, tools: State<K>, key: K) =>
  136. Effect.gen(function* () {
  137. const tool = tools[key]
  138. if (!tool) return { tools }
  139. return { tools: withoutTool(tools, key), event: yield* toolCall(route, tool) }
  140. })
  141. /**
  142. * Finalize one pending tool call with an authoritative final input string.
  143. * OpenAI Responses can send accumulated deltas and then repeat the completed
  144. * arguments on `response.output_item.done`; the final value wins.
  145. */
  146. export const finishWithInput = <K extends StreamKey>(route: string, tools: State<K>, key: K, input: string) =>
  147. Effect.gen(function* () {
  148. const tool = tools[key]
  149. if (!tool) return { tools }
  150. return { tools: withoutTool(tools, key), event: yield* toolCall(route, tool, input) }
  151. })
  152. /**
  153. * Finalize every pending tool call at once. OpenAI Chat has this shape: it does
  154. * not emit per-tool stop events, so all accumulated calls finish when the choice
  155. * receives a terminal `finish_reason`.
  156. */
  157. export const finishAll = <K extends StreamKey>(route: string, tools: State<K>) =>
  158. Effect.gen(function* () {
  159. const pending = Object.values<PendingTool | undefined>(tools).filter(
  160. (tool): tool is PendingTool => tool !== undefined,
  161. )
  162. return {
  163. tools: empty<K>(),
  164. events: yield* Effect.forEach(pending, (tool) => toolCall(route, tool)),
  165. }
  166. })
  167. export * as ToolStream from "./tool-stream"