shared.ts 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317
  1. import { Buffer } from "node:buffer"
  2. import { Effect, JsonSchema, Schema, Stream } from "effect"
  3. import * as Sse from "effect/unstable/encoding/Sse"
  4. import { Headers, HttpClientRequest } from "effect/unstable/http"
  5. import {
  6. InvalidProviderOutputReason,
  7. InvalidRequestReason,
  8. LLMError,
  9. type ContentPart,
  10. type LLMRequest,
  11. type MediaPart,
  12. type TextPart,
  13. type ToolResultPart,
  14. } from "../schema"
  15. import { isRecord } from "../utils/record"
  16. export { isRecord }
  17. export const Json = Schema.fromJsonString(Schema.Unknown)
  18. export const decodeJson = Schema.decodeUnknownSync(Json)
  19. export const encodeJson = Schema.encodeSync(Json)
  20. export const JsonObject = Schema.Record(Schema.String, Schema.Unknown)
  21. export const optionalArray = <const S extends Schema.Top>(schema: S) => Schema.optional(Schema.Array(schema))
  22. export const optionalNull = <const S extends Schema.Top>(schema: S) => Schema.optional(Schema.NullOr(schema))
  23. /** OpenAI function schemas require one flat object at the top level. */
  24. export const openAiToolInputSchema = (schema: JsonSchema.JsonSchema): JsonSchema.JsonSchema => {
  25. const variants = Array.isArray(schema.anyOf) ? schema.anyOf.filter(isRecord) : []
  26. const flattened =
  27. variants.length === 0
  28. ? { ...schema, type: "object" }
  29. : {
  30. ...Object.fromEntries(Object.entries(schema).filter(([key]) => key !== "anyOf")),
  31. type: "object",
  32. properties: variants.reduce(
  33. (properties, variant) => ({ ...(isRecord(variant.properties) ? variant.properties : {}), ...properties }),
  34. {},
  35. ),
  36. additionalProperties: false,
  37. }
  38. const normalized = removeNullSchemas(flattened)
  39. return isRecord(normalized) ? normalized : { type: "object" }
  40. }
  41. const removeNullSchemas = (value: unknown): unknown => {
  42. if (Array.isArray(value)) return value.map(removeNullSchemas)
  43. if (!isRecord(value)) return value
  44. const fields = Object.fromEntries(
  45. Object.entries(value)
  46. .filter(([key]) => key !== "anyOf")
  47. .map(([key, field]) => [key, removeNullSchemas(field)]),
  48. )
  49. if (!Array.isArray(value.anyOf)) return fields
  50. const variants = value.anyOf.filter((variant) => !isRecord(variant) || variant.type !== "null").map(removeNullSchemas)
  51. if (variants.length === 1 && isRecord(variants[0])) return { ...fields, ...variants[0] }
  52. return { ...fields, anyOf: variants }
  53. }
  54. /**
  55. * Streaming tool-call accumulator. Adapters that build a tool call across
  56. * multiple `tool-input-delta` chunks store the partial JSON input string here
  57. * and finalize it with `parseToolInput` once the call completes.
  58. */
  59. export interface ToolAccumulator {
  60. readonly id: string
  61. readonly name: string
  62. readonly input: string
  63. }
  64. /**
  65. * `Usage.totalTokens` policy shared by every route. Honors a provider-
  66. * supplied total; otherwise falls back to `inputTokens + outputTokens` only
  67. * when at least one is defined. Returns `undefined` when neither input nor
  68. * output is known so routes don't publish a misleading `0`.
  69. *
  70. * Under the additive `LLM.Usage` contract, `inputTokens` and `outputTokens`
  71. * are the non-cached input and visible output only. The provider-supplied
  72. * `total` is the source of truth when present; the computed fallback
  73. * under-counts cache and reasoning by design and exists mainly so
  74. * Anthropic-style providers (which don't surface a total) still get a
  75. * sensible aggregate on the input + output axes.
  76. */
  77. export const totalTokens = (
  78. inputTokens: number | undefined,
  79. outputTokens: number | undefined,
  80. total: number | undefined,
  81. ) => {
  82. if (total !== undefined) return total
  83. if (inputTokens === undefined && outputTokens === undefined) return undefined
  84. return (inputTokens ?? 0) + (outputTokens ?? 0)
  85. }
  86. /**
  87. * Subtract `subtrahend` from `total`, clamping to zero if the provider
  88. * reports a non-sensical breakdown (e.g. `cached_tokens > prompt_tokens`).
  89. * Used by protocol mappers when deriving a non-overlapping breakdown field
  90. * from a provider's inclusive total — `nonCachedInputTokens` from
  91. * `inputTokens - cacheReadInputTokens - cacheWriteInputTokens`.
  92. *
  93. * If `total` is `undefined`, returns `undefined` (we don't fabricate
  94. * counts). If `subtrahend` is `undefined`, returns `total` unchanged. The
  95. * provider-native breakdown stays available on `Usage.native` for debugging.
  96. */
  97. export const subtractTokens = (total: number | undefined, subtrahend: number | undefined): number | undefined => {
  98. if (total === undefined) return undefined
  99. if (subtrahend === undefined) return total
  100. return Math.max(0, total - subtrahend)
  101. }
  102. /**
  103. * Sum a list of optional token counts, returning `undefined` only when
  104. * every value is `undefined` (so we don't fabricate a `0`). Used by
  105. * protocol mappers to derive the inclusive `inputTokens` total from a
  106. * provider that natively reports a non-overlapping breakdown
  107. * (e.g. Anthropic, whose `input_tokens` is already non-cached only).
  108. */
  109. export const sumTokens = (...values: ReadonlyArray<number | undefined>): number | undefined => {
  110. if (values.every((value) => value === undefined)) return undefined
  111. return values.reduce((acc: number, value) => acc + (value ?? 0), 0)
  112. }
  113. export const eventError = (route: string, message: string, raw?: string) =>
  114. new LLMError({
  115. module: "ProviderShared",
  116. method: "stream",
  117. reason: new InvalidProviderOutputReason({ route, message, raw }),
  118. })
  119. export const parseJson = (route: string, input: string, message: string) =>
  120. Effect.try({
  121. try: () => decodeJson(input),
  122. catch: () => eventError(route, message, input),
  123. })
  124. /**
  125. * Join the `text` field of a list of parts with newlines. Used by routes
  126. * that flatten system / message content arrays into a single provider string
  127. * (OpenAI Chat `system` content, OpenAI Responses `system` content, Gemini
  128. * `systemInstruction.parts[].text`).
  129. */
  130. export const joinText = (parts: ReadonlyArray<{ readonly text: string }>) => parts.map((part) => part.text).join("\n")
  131. const escapeSystemUpdateText = (text: string) =>
  132. text.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;")
  133. /**
  134. * Stable fallback representation for chronological `Message.system(...)`
  135. * updates on routes that do not support that privileged role natively. The
  136. * wrapper remains visibly lower-authority user text, preserves the original
  137. * temporal position, and XML-escapes content so it cannot close the wrapper.
  138. */
  139. export const wrapSystemUpdate = (parts: ReadonlyArray<{ readonly text: string }>) =>
  140. `<system-update>\n${escapeSystemUpdateText(joinText(parts))}\n</system-update>`
  141. /**
  142. * Chronological system updates deliberately accept text only. Do not insert
  143. * raw retrieved, tool, or web content into privileged updates: keep untrusted
  144. * data in ordinary user/tool messages instead.
  145. */
  146. export const systemUpdateText = Effect.fn("ProviderShared.systemUpdateText")(function* (
  147. route: string,
  148. message: LLMRequest["messages"][number],
  149. ) {
  150. const content: TextPart[] = []
  151. for (const part of message.content) {
  152. if (!supportsContent(part, ["text"])) return yield* unsupportedContent(route, "system", ["text"])
  153. content.push(part)
  154. }
  155. return content
  156. })
  157. /** Lower an unsupported privileged update into visible, in-order user text. */
  158. export const wrappedSystemUpdate = Effect.fn("ProviderShared.wrappedSystemUpdate")(function* (
  159. route: string,
  160. message: LLMRequest["messages"][number],
  161. ) {
  162. const content = yield* systemUpdateText(route, message)
  163. return { type: "text" as const, text: wrapSystemUpdate(content), cache: content.at(-1)?.cache }
  164. })
  165. /**
  166. * Parse the streamed JSON input of a tool call. Treats an empty string as
  167. * `"{}"` — providers occasionally finish a tool call without ever emitting
  168. * input deltas (e.g. zero-arg tools). The error message is uniform across
  169. * routes: `Invalid JSON input for <route> tool call <name>`.
  170. */
  171. export const parseToolInput = (route: string, name: string, raw: string) =>
  172. parseJson(route, raw || "{}", `Invalid JSON input for ${route} tool call ${name}`)
  173. /**
  174. * Encode a `MediaPart`'s raw bytes for inclusion in a JSON request body.
  175. * `data: string` is assumed to already be base64 (matches caller convention
  176. * across Gemini / Bedrock); `data: Uint8Array` is base64-encoded here. Used
  177. * by every route that supports image / document inputs.
  178. */
  179. export const mediaBytes = (part: MediaPart) =>
  180. typeof part.data === "string" ? part.data : Buffer.from(part.data).toString("base64")
  181. export const mediaBase64 = (part: MediaPart) => {
  182. if (typeof part.data !== "string" || !part.data.startsWith("data:")) return mediaBytes(part)
  183. return part.data.slice(part.data.indexOf(",") + 1)
  184. }
  185. export const mediaDataUrl = (part: MediaPart) =>
  186. typeof part.data === "string" && part.data.startsWith("data:")
  187. ? part.data
  188. : `data:${part.mediaType};base64,${mediaBytes(part)}`
  189. export const trimBaseUrl = (value: string) => value.replace(/\/+$/, "")
  190. export const toolResultText = (part: ToolResultPart) => {
  191. if (part.result.type === "text" || part.result.type === "error") return String(part.result.value)
  192. if (part.result.type === "content") return encodeJson(part.result.value)
  193. return encodeJson(part.result.value)
  194. }
  195. export const errorText = (error: unknown) => {
  196. if (error instanceof Error) return error.message
  197. if (typeof error === "string") return error
  198. if (typeof error === "number" || typeof error === "boolean" || typeof error === "bigint") return String(error)
  199. if (error === null) return "null"
  200. if (error === undefined) return "undefined"
  201. return "Unknown stream error"
  202. }
  203. /**
  204. * `framing` step for Server-Sent Events. Decodes UTF-8, runs the SSE channel
  205. * decoder, and drops empty / `[DONE]` keep-alive events so the downstream
  206. * `decodeChunk` sees one JSON string per element. The SSE channel emits a
  207. * `Retry` control event on its error channel; we drop it here (we don't
  208. * implement client-driven retries) so the public error channel stays
  209. * `LLMError`.
  210. */
  211. export const sseFraming = (bytes: Stream.Stream<Uint8Array, LLMError>): Stream.Stream<string, LLMError> =>
  212. bytes.pipe(
  213. Stream.decodeText(),
  214. Stream.pipeThroughChannel(Sse.decode()),
  215. Stream.catchTag("Retry", () => Stream.empty),
  216. Stream.filter((event) => event.data.length > 0 && event.data !== "[DONE]"),
  217. Stream.map((event) => event.data),
  218. )
  219. /**
  220. * Canonical invalid-request constructor. Lift one-line `const invalid =
  221. * (message) => invalidRequest(message)` aliases out of every
  222. * route so the error constructor lives in one place. If we ever extend
  223. * `InvalidRequestReason` with route context or trace metadata, the change
  224. * lands here.
  225. */
  226. export const invalidRequest = (message: string) =>
  227. new LLMError({
  228. module: "ProviderShared",
  229. method: "request",
  230. reason: new InvalidRequestReason({ message }),
  231. })
  232. export const matchToolChoice = <Auto, None, Required, Tool>(
  233. route: string,
  234. toolChoice: NonNullable<LLMRequest["toolChoice"]>,
  235. cases: {
  236. readonly auto: () => Auto
  237. readonly none: () => None
  238. readonly required: () => Required
  239. readonly tool: (name: string) => Tool
  240. },
  241. ) =>
  242. Effect.gen(function* () {
  243. if (toolChoice.type === "auto") return cases.auto()
  244. if (toolChoice.type === "none") return cases.none()
  245. if (toolChoice.type === "required") return cases.required()
  246. if (!toolChoice.name) return yield* invalidRequest(`${route} tool choice requires a tool name`)
  247. return cases.tool(toolChoice.name)
  248. })
  249. type ContentType = ContentPart["type"]
  250. const formatContentTypes = (types: ReadonlyArray<ContentType>) => {
  251. if (types.length <= 1) return types[0] ?? ""
  252. if (types.length === 2) return `${types[0]} and ${types[1]}`
  253. return `${types.slice(0, -1).join(", ")}, and ${types.at(-1)}`
  254. }
  255. export const supportsContent = <const Type extends ContentType>(
  256. part: ContentPart,
  257. types: ReadonlyArray<Type>,
  258. ): part is Extract<ContentPart, { readonly type: Type }> => (types as ReadonlyArray<ContentType>).includes(part.type)
  259. export const unsupportedContent = (
  260. route: string,
  261. role: LLMRequest["messages"][number]["role"],
  262. types: ReadonlyArray<ContentType>,
  263. ) => invalidRequest(`${route} ${role} messages only support ${formatContentTypes(types)} content for now`)
  264. /**
  265. * Build a `validate` step from a Schema decoder. Replaces the per-route
  266. * lambda body `(payload) => decode(payload).pipe(Effect.mapError((e) =>
  267. * invalid(e.message)))`. Any decode error is translated into
  268. * `LLMError` carrying the original parse-error message.
  269. */
  270. export const validateWith =
  271. <A, I, E extends { readonly message: string }>(decode: (input: I) => Effect.Effect<A, E>) =>
  272. (payload: I) =>
  273. decode(payload).pipe(Effect.mapError((error) => invalidRequest(error.message)))
  274. /**
  275. * Build an HTTP POST with a JSON body. Sets `content-type: application/json`
  276. * automatically after caller-supplied headers so routes cannot accidentally
  277. * send JSON with a stale content type. The body is passed pre-encoded so
  278. * routes can choose between
  279. * `Schema.encodeSync(payload)` and `ProviderShared.encodeJson(payload)`.
  280. */
  281. export const jsonPost = (input: { readonly url: string; readonly body: string; readonly headers?: Headers.Input }) =>
  282. HttpClientRequest.post(input.url).pipe(
  283. HttpClientRequest.setHeaders(Headers.set(Headers.fromInput(input.headers), "content-type", "application/json")),
  284. HttpClientRequest.bodyText(input.body, "application/json"),
  285. )
  286. export * as ProviderShared from "./shared"