shared.ts 8.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203
  1. import { Buffer } from "node:buffer"
  2. import { Effect, Schema, Stream } from "effect"
  3. import * as Sse from "effect/unstable/encoding/Sse"
  4. import { Headers, HttpClientRequest } from "effect/unstable/http"
  5. import {
  6. InvalidProviderOutputReason,
  7. InvalidRequestReason,
  8. LLMError,
  9. type ContentPart,
  10. type LLMRequest,
  11. type MediaPart,
  12. type ToolResultPart,
  13. } from "../schema"
  14. export const Json = Schema.fromJsonString(Schema.Unknown)
  15. export const decodeJson = Schema.decodeUnknownSync(Json)
  16. export const encodeJson = Schema.encodeSync(Json)
  17. export const JsonObject = Schema.Record(Schema.String, Schema.Unknown)
  18. export const optionalArray = <const S extends Schema.Top>(schema: S) => Schema.optional(Schema.Array(schema))
  19. export const optionalNull = <const S extends Schema.Top>(schema: S) => Schema.optional(Schema.NullOr(schema))
  20. /**
  21. * Plain-record narrowing. Excludes arrays so routes checking nested JSON
  22. * Schema fragments don't accidentally treat a tuple as a key/value bag.
  23. */
  24. export const isRecord = (value: unknown): value is Record<string, unknown> =>
  25. typeof value === "object" && value !== null && !Array.isArray(value)
  26. /**
  27. * Streaming tool-call accumulator. Adapters that build a tool call across
  28. * multiple `tool-input-delta` chunks store the partial JSON input string here
  29. * and finalize it with `parseToolInput` once the call completes.
  30. */
  31. export interface ToolAccumulator {
  32. readonly id: string
  33. readonly name: string
  34. readonly input: string
  35. }
  36. /**
  37. * `Usage.totalTokens` policy shared by every route. Honors a provider-
  38. * supplied total; otherwise falls back to `inputTokens + outputTokens` only
  39. * when at least one is defined. Returns `undefined` when neither input nor
  40. * output is known so routes don't publish a misleading `0`.
  41. */
  42. export const totalTokens = (
  43. inputTokens: number | undefined,
  44. outputTokens: number | undefined,
  45. total: number | undefined,
  46. ) => {
  47. if (total !== undefined) return total
  48. if (inputTokens === undefined && outputTokens === undefined) return undefined
  49. return (inputTokens ?? 0) + (outputTokens ?? 0)
  50. }
  51. export const eventError = (route: string, message: string, raw?: string) =>
  52. new LLMError({
  53. module: "ProviderShared",
  54. method: "stream",
  55. reason: new InvalidProviderOutputReason({ route, message, raw }),
  56. })
  57. export const parseJson = (route: string, input: string, message: string) =>
  58. Effect.try({
  59. try: () => decodeJson(input),
  60. catch: () => eventError(route, message, input),
  61. })
  62. /**
  63. * Join the `text` field of a list of parts with newlines. Used by routes
  64. * that flatten system / message content arrays into a single provider string
  65. * (OpenAI Chat `system` content, OpenAI Responses `system` content, Gemini
  66. * `systemInstruction.parts[].text`).
  67. */
  68. export const joinText = (parts: ReadonlyArray<{ readonly text: string }>) => parts.map((part) => part.text).join("\n")
  69. /**
  70. * Parse the streamed JSON input of a tool call. Treats an empty string as
  71. * `"{}"` — providers occasionally finish a tool call without ever emitting
  72. * input deltas (e.g. zero-arg tools). The error message is uniform across
  73. * routes: `Invalid JSON input for <route> tool call <name>`.
  74. */
  75. export const parseToolInput = (route: string, name: string, raw: string) =>
  76. parseJson(route, raw || "{}", `Invalid JSON input for ${route} tool call ${name}`)
  77. /**
  78. * Encode a `MediaPart`'s raw bytes for inclusion in a JSON request body.
  79. * `data: string` is assumed to already be base64 (matches caller convention
  80. * across Gemini / Bedrock); `data: Uint8Array` is base64-encoded here. Used
  81. * by every route that supports image / document inputs.
  82. */
  83. export const mediaBytes = (part: MediaPart) =>
  84. typeof part.data === "string" ? part.data : Buffer.from(part.data).toString("base64")
  85. export const trimBaseUrl = (value: string) => value.replace(/\/+$/, "")
  86. export const toolResultText = (part: ToolResultPart) => {
  87. if (part.result.type === "text" || part.result.type === "error") return String(part.result.value)
  88. return encodeJson(part.result.value)
  89. }
  90. export const errorText = (error: unknown) => {
  91. if (error instanceof Error) return error.message
  92. if (typeof error === "string") return error
  93. if (typeof error === "number" || typeof error === "boolean" || typeof error === "bigint") return String(error)
  94. if (error === null) return "null"
  95. if (error === undefined) return "undefined"
  96. return "Unknown stream error"
  97. }
  98. /**
  99. * `framing` step for Server-Sent Events. Decodes UTF-8, runs the SSE channel
  100. * decoder, and drops empty / `[DONE]` keep-alive events so the downstream
  101. * `decodeChunk` sees one JSON string per element. The SSE channel emits a
  102. * `Retry` control event on its error channel; we drop it here (we don't
  103. * implement client-driven retries) so the public error channel stays
  104. * `LLMError`.
  105. */
  106. export const sseFraming = (bytes: Stream.Stream<Uint8Array, LLMError>): Stream.Stream<string, LLMError> =>
  107. bytes.pipe(
  108. Stream.decodeText(),
  109. Stream.pipeThroughChannel(Sse.decode()),
  110. Stream.catchTag("Retry", () => Stream.empty),
  111. Stream.filter((event) => event.data.length > 0 && event.data !== "[DONE]"),
  112. Stream.map((event) => event.data),
  113. )
  114. /**
  115. * Canonical invalid-request constructor. Lift one-line `const invalid =
  116. * (message) => invalidRequest(message)` aliases out of every
  117. * route so the error constructor lives in one place. If we ever extend
  118. * `InvalidRequestReason` with route context or trace metadata, the change
  119. * lands here.
  120. */
  121. export const invalidRequest = (message: string) =>
  122. new LLMError({
  123. module: "ProviderShared",
  124. method: "request",
  125. reason: new InvalidRequestReason({ message }),
  126. })
  127. export const matchToolChoice = <Auto, None, Required, Tool>(
  128. route: string,
  129. toolChoice: NonNullable<LLMRequest["toolChoice"]>,
  130. cases: {
  131. readonly auto: () => Auto
  132. readonly none: () => None
  133. readonly required: () => Required
  134. readonly tool: (name: string) => Tool
  135. },
  136. ) =>
  137. Effect.gen(function* () {
  138. if (toolChoice.type === "auto") return cases.auto()
  139. if (toolChoice.type === "none") return cases.none()
  140. if (toolChoice.type === "required") return cases.required()
  141. if (!toolChoice.name) return yield* invalidRequest(`${route} tool choice requires a tool name`)
  142. return cases.tool(toolChoice.name)
  143. })
  144. type ContentType = ContentPart["type"]
  145. const formatContentTypes = (types: ReadonlyArray<ContentType>) => {
  146. if (types.length <= 1) return types[0] ?? ""
  147. if (types.length === 2) return `${types[0]} and ${types[1]}`
  148. return `${types.slice(0, -1).join(", ")}, and ${types.at(-1)}`
  149. }
  150. export const supportsContent = <const Type extends ContentType>(
  151. part: ContentPart,
  152. types: ReadonlyArray<Type>,
  153. ): part is Extract<ContentPart, { readonly type: Type }> => (types as ReadonlyArray<ContentType>).includes(part.type)
  154. export const unsupportedContent = (
  155. route: string,
  156. role: LLMRequest["messages"][number]["role"],
  157. types: ReadonlyArray<ContentType>,
  158. ) => invalidRequest(`${route} ${role} messages only support ${formatContentTypes(types)} content for now`)
  159. /**
  160. * Build a `validate` step from a Schema decoder. Replaces the per-route
  161. * lambda body `(payload) => decode(payload).pipe(Effect.mapError((e) =>
  162. * invalid(e.message)))`. Any decode error is translated into
  163. * `LLMError` carrying the original parse-error message.
  164. */
  165. export const validateWith =
  166. <A, I, E extends { readonly message: string }>(decode: (input: I) => Effect.Effect<A, E>) =>
  167. (payload: I) =>
  168. decode(payload).pipe(Effect.mapError((error) => invalidRequest(error.message)))
  169. /**
  170. * Build an HTTP POST with a JSON body. Sets `content-type: application/json`
  171. * automatically after caller-supplied headers so routes cannot accidentally
  172. * send JSON with a stale content type. The body is passed pre-encoded so
  173. * routes can choose between
  174. * `Schema.encodeSync(payload)` and `ProviderShared.encodeJson(payload)`.
  175. */
  176. export const jsonPost = (input: { readonly url: string; readonly body: string; readonly headers?: Headers.Input }) =>
  177. HttpClientRequest.post(input.url).pipe(
  178. HttpClientRequest.setHeaders(Headers.set(Headers.fromInput(input.headers), "content-type", "application/json")),
  179. HttpClientRequest.bodyText(input.body, "application/json"),
  180. )
  181. export * as ProviderShared from "./shared"