client.ts 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436
  1. import { Cause, Context, Effect, Layer, Schema, Stream } from "effect"
  2. import * as Option from "effect/Option"
  3. import { Auth, type Auth as AuthDef } from "./auth"
  4. import { Endpoint, type EndpointPatch } from "./endpoint"
  5. import { RequestExecutor } from "./executor"
  6. import type { Framing } from "./framing"
  7. import { HttpTransport } from "./transport"
  8. import type { Transport, TransportRuntime } from "./transport"
  9. import { WebSocketExecutor } from "./transport"
  10. import type { Protocol } from "./protocol"
  11. import { applyCachePolicy } from "../cache-policy"
  12. import * as ProviderShared from "../protocols/shared"
  13. import type { LLMError, LLMEvent, PreparedRequestOf, ProtocolID, ProviderOptions } from "../schema"
  14. import {
  15. GenerationOptions,
  16. HttpOptions,
  17. LLMRequest,
  18. LLMResponse,
  19. Model,
  20. ModelLimits,
  21. LLMError as LLMErrorClass,
  22. PreparedRequest,
  23. ProviderID,
  24. mergeGenerationOptions,
  25. mergeHttpOptions,
  26. mergeProviderOptions,
  27. } from "../schema"
  28. export interface RouteBody<Body> {
  29. /** Schema for the validated provider-native body sent as the JSON request. */
  30. readonly schema: Schema.Codec<Body, unknown>
  31. /** Build the provider-native body from a common `LLMRequest`. */
  32. readonly from: (request: LLMRequest) => Effect.Effect<Body, LLMError>
  33. }
  34. export interface Route<Body, Prepared = unknown> {
  35. readonly id: string
  36. readonly provider?: ProviderID
  37. readonly protocol: ProtocolID
  38. readonly endpoint: Endpoint<Body>
  39. readonly auth: AuthDef
  40. readonly transport: Transport<Body, Prepared, unknown>
  41. readonly defaults: RouteDefaults
  42. readonly body: RouteBody<Body>
  43. readonly with: (patch: RoutePatch<Body, Prepared>) => Route<Body, Prepared>
  44. readonly model: (input: RouteMappedModelInput) => Model
  45. readonly prepareTransport: (body: Body, request: LLMRequest) => Effect.Effect<Prepared, LLMError>
  46. readonly streamPrepared: (
  47. prepared: Prepared,
  48. request: LLMRequest,
  49. runtime: TransportRuntime,
  50. ) => Stream.Stream<LLMEvent, LLMError>
  51. }
  52. // Route registries intentionally erase body generics after construction.
  53. // Normal call sites use `OpenAIChat.route`; callers only need body types
  54. // when preparing a request with a protocol-specific type assertion.
  55. // oxlint-disable-next-line typescript-eslint/no-explicit-any
  56. export type AnyRoute = Route<any, any>
  57. export type HttpOptionsInput = HttpOptions.Input
  58. export type RouteModelInput = Omit<Model.Input, "provider" | "route">
  59. export type RouteRoutedModelInput = Omit<Model.Input, "route">
  60. export interface RouteDefaults {
  61. readonly headers?: Record<string, string>
  62. readonly limits?: ModelLimits
  63. readonly generation?: GenerationOptions
  64. readonly providerOptions?: ProviderOptions
  65. readonly http?: HttpOptions
  66. }
  67. export interface RouteDefaultsInput {
  68. readonly headers?: Record<string, string>
  69. readonly limits?: ModelLimits.Input
  70. readonly generation?: GenerationOptions.Input
  71. readonly providerOptions?: ProviderOptions
  72. readonly http?: HttpOptions.Input
  73. }
  74. export interface RoutePatch<Body, Prepared> extends RouteDefaultsInput {
  75. readonly id?: string
  76. readonly provider?: string | ProviderID
  77. readonly auth?: AuthDef
  78. readonly transport?: Transport<Body, Prepared, unknown>
  79. readonly endpoint?: EndpointPatch<Body>
  80. }
  81. type RouteMappedModelInput = RouteModelInput | RouteRoutedModelInput
  82. const makeRouteModel = (route: AnyRoute, mapped: RouteMappedModelInput) => {
  83. const provider = route.provider ?? ("provider" in mapped ? mapped.provider : undefined)
  84. if (!provider) throw new Error(`Route.model(${route.id}) requires a provider`)
  85. if (!endpointBaseURL(route.endpoint))
  86. throw new Error(`Route.model(${route.id}) requires an endpoint baseURL — configure it on the route first`)
  87. return Model.make({
  88. ...mapped,
  89. provider,
  90. route,
  91. })
  92. }
  93. const mergeRouteDefaults = (base: RouteDefaults | undefined, patch: RouteDefaultsInput): RouteDefaults => {
  94. const headers = mergeHeaders(base?.headers, patch.headers)
  95. return {
  96. ...base,
  97. ...patch,
  98. headers,
  99. limits: patch.limits === undefined ? base?.limits : ModelLimits.make(patch.limits),
  100. generation: mergeGenerationOptions(generationOptions(base?.generation), generationOptions(patch.generation)),
  101. providerOptions: mergeProviderOptions(base?.providerOptions, patch.providerOptions),
  102. http: mergeHttpOptions(
  103. base?.http,
  104. httpOptions(patch.http),
  105. headers === undefined ? undefined : new HttpOptions({ headers }),
  106. ),
  107. }
  108. }
  109. const endpointBaseURL = <Body>(endpoint: Endpoint<Body>) =>
  110. typeof endpoint.baseURL === "string" ? endpoint.baseURL : undefined
  111. const mergeHeaders = (...items: ReadonlyArray<Record<string, string> | undefined>) => {
  112. const entries = items.flatMap((item) =>
  113. item === undefined ? [] : Object.entries(item).filter((entry): entry is [string, string] => entry[1] !== undefined),
  114. )
  115. if (entries.length === 0) return undefined
  116. return Object.fromEntries(entries)
  117. }
  118. export const generationOptions = (input: GenerationOptions.Input | undefined) =>
  119. input === undefined ? undefined : GenerationOptions.make(input)
  120. export const httpOptions = (input: HttpOptionsInput | undefined) => {
  121. if (input === undefined) return input
  122. return HttpOptions.make(input)
  123. }
  124. export interface Interface {
  125. /**
  126. * Compile a request through protocol body construction, validation, and HTTP
  127. * preparation without sending it. Returns the prepared request including the
  128. * provider-native body.
  129. *
  130. * Pass a `Body` type argument to statically expose the route's body
  131. * shape (e.g. `prepare<OpenAIChatBody>(...)`) — the runtime body is
  132. * identical, so this is a type-level assertion the caller makes about which
  133. * route the request will resolve to.
  134. */
  135. readonly prepare: <Body = unknown>(request: LLMRequest) => Effect.Effect<PreparedRequestOf<Body>, LLMError>
  136. readonly stream: StreamMethod
  137. readonly generate: GenerateMethod
  138. }
  139. export interface StreamMethod {
  140. (request: LLMRequest): Stream.Stream<LLMEvent, LLMError>
  141. }
  142. export interface GenerateMethod {
  143. (request: LLMRequest): Effect.Effect<LLMResponse, LLMError>
  144. }
  145. export class Service extends Context.Service<Service, Interface>()("@opencode/LLMClient") {}
  146. const resolveRequestOptions = (request: LLMRequest) => {
  147. const routeDefaults = request.model.route.defaults
  148. const modelDefaults = request.model.defaults
  149. const generation = mergeGenerationOptions(routeDefaults.generation, modelDefaults?.generation, request.generation)
  150. return LLMRequest.update(request, {
  151. generation: generation ?? new GenerationOptions({}),
  152. providerOptions: mergeProviderOptions(
  153. routeDefaults.providerOptions,
  154. modelDefaults?.providerOptions,
  155. request.providerOptions,
  156. ),
  157. http: mergeHttpOptions(routeDefaults.http, modelDefaults?.http, request.http),
  158. })
  159. }
  160. export interface MakeInput<Body, Frame, Event, State> {
  161. /** Route id used in diagnostics and prepared request metadata. */
  162. readonly id: string
  163. /** Provider identity for route-owned model construction. */
  164. readonly provider?: string | ProviderID
  165. /** Semantic API contract — owns body construction, body schema, and parsing. */
  166. readonly protocol: Protocol<Body, Frame, Event, State>
  167. /** Where the request is sent. */
  168. readonly endpoint: Endpoint<Body>
  169. /** Per-request transport auth. Provider facades override this via `route.with(...)`. */
  170. readonly auth?: AuthDef
  171. /** Stream framing — bytes -> frames before `protocol.stream.event` decoding. */
  172. readonly framing: Framing<Frame>
  173. /** Static / per-request headers added before `auth` runs. */
  174. readonly headers?: (input: { readonly request: LLMRequest }) => Record<string, string>
  175. /** Route/request defaults used when compiling requests for this route. */
  176. readonly defaults?: RouteDefaultsInput
  177. }
  178. export interface MakeTransportInput<Body, Prepared, Frame, Event, State> {
  179. /** Route id used in diagnostics and prepared request metadata. */
  180. readonly id: string
  181. /** Provider identity for route-owned model construction. */
  182. readonly provider?: string | ProviderID
  183. /** Semantic API contract — owns body construction, body schema, and parsing. */
  184. readonly protocol: Protocol<Body, Frame, Event, State>
  185. /** Where the request is sent. */
  186. readonly endpoint: Endpoint<Body>
  187. /** Per-request transport auth. Provider facades override this via `route.with(...)`. */
  188. readonly auth?: AuthDef
  189. /** Static / per-request headers added before `auth` runs. */
  190. readonly headers?: (input: { readonly request: LLMRequest }) => Record<string, string>
  191. /** Runnable transport route. */
  192. readonly transport: Transport<Body, Prepared, Frame>
  193. /** Route/request defaults used when compiling requests for this route. */
  194. readonly defaults?: RouteDefaultsInput
  195. }
  196. const streamError = (route: string, message: string, cause: Cause.Cause<unknown>) => {
  197. const failed = cause.reasons.find(Cause.isFailReason)?.error
  198. if (failed instanceof LLMErrorClass) return failed
  199. return ProviderShared.eventError(route, message, Cause.pretty(cause))
  200. }
  201. function makeFromTransport<Body, Prepared, Frame, Event, State>(
  202. input: MakeTransportInput<Body, Prepared, Frame, Event, State>,
  203. ): Route<Body, Prepared> {
  204. const protocol = input.protocol
  205. const encodeBody = Schema.encodeSync(Schema.fromJsonString(protocol.body.schema))
  206. const decodeEventEffect = Schema.decodeUnknownEffect(protocol.stream.event)
  207. const decodeEvent = (route: string) => (frame: Frame) =>
  208. decodeEventEffect(frame).pipe(
  209. Effect.mapError(() =>
  210. ProviderShared.eventError(
  211. input.id,
  212. `Invalid ${route} stream event`,
  213. typeof frame === "string" ? frame : ProviderShared.encodeJson(frame),
  214. ),
  215. ),
  216. )
  217. type BuiltRouteInput = Omit<MakeTransportInput<Body, Prepared, Frame, Event, State>, "defaults"> & {
  218. readonly defaults?: RouteDefaults
  219. }
  220. const build = (routeInput: BuiltRouteInput): Route<Body, Prepared> => {
  221. const route: Route<Body, Prepared> = {
  222. id: routeInput.id,
  223. provider: routeInput.provider === undefined ? undefined : ProviderID.make(routeInput.provider),
  224. protocol: protocol.id,
  225. endpoint: routeInput.endpoint,
  226. auth: routeInput.auth ?? Auth.none,
  227. transport: routeInput.transport,
  228. defaults: routeInput.defaults ?? {},
  229. body: protocol.body,
  230. with: (patch: RoutePatch<Body, Prepared>) => {
  231. const { id, provider, auth, transport, endpoint, ...defaults } = patch
  232. return build({
  233. ...routeInput,
  234. id: id ?? routeInput.id,
  235. provider: provider ?? routeInput.provider,
  236. auth: auth ?? routeInput.auth,
  237. endpoint: endpoint ? Endpoint.merge(routeInput.endpoint, endpoint) : routeInput.endpoint,
  238. transport: (transport as Transport<Body, Prepared, Frame> | undefined) ?? routeInput.transport,
  239. defaults: mergeRouteDefaults(route.defaults, defaults),
  240. })
  241. },
  242. model: (input) => makeRouteModel(route, input),
  243. prepareTransport: (body, request) =>
  244. routeInput.transport.prepare({
  245. body,
  246. request,
  247. endpoint: routeInput.endpoint,
  248. auth: routeInput.auth ?? Auth.none,
  249. encodeBody,
  250. headers: routeInput.headers,
  251. }),
  252. streamPrepared: (prepared: Prepared, request: LLMRequest, runtime: TransportRuntime) => {
  253. const route = `${request.model.provider}/${request.model.route.id}`
  254. const events = routeInput.transport
  255. .frames(prepared, request, runtime)
  256. .pipe(
  257. Stream.mapEffect(decodeEvent(route)),
  258. protocol.stream.terminal ? Stream.takeUntil(protocol.stream.terminal) : (stream) => stream,
  259. )
  260. return events.pipe(
  261. Stream.mapAccumEffect(
  262. () => protocol.stream.initial(request),
  263. protocol.stream.step,
  264. protocol.stream.onHalt ? { onHalt: protocol.stream.onHalt } : undefined,
  265. ),
  266. Stream.catchCause((cause) => Stream.fail(streamError(route, `Failed to read ${route} stream`, cause))),
  267. )
  268. },
  269. } satisfies Route<Body, Prepared>
  270. return route
  271. }
  272. return build({ ...input, defaults: mergeRouteDefaults(undefined, input.defaults ?? {}) })
  273. }
  274. export function make<Body, Prepared, Frame, Event, State>(
  275. input: MakeTransportInput<Body, Prepared, Frame, Event, State>,
  276. ): Route<Body, Prepared>
  277. /**
  278. * Build a `Route` by composing the four orthogonal pieces of a deployment:
  279. *
  280. * - `Protocol` — what is the API I'm speaking?
  281. * - `Endpoint` — where do I send the request?
  282. * - `Auth` — how do I authenticate it?
  283. * - `Framing` — how do I cut the response stream into protocol frames?
  284. *
  285. * Plus optional `headers` for cross-cutting deployment concerns (provider
  286. * version pins, per-deployment quirks).
  287. *
  288. * This is the canonical route constructor. If a new route does not fit
  289. * this four-axis model, add a purpose-built constructor rather than widening
  290. * the public surface preemptively.
  291. */
  292. export function make<Body, Frame, Event, State>(
  293. input: MakeInput<Body, Frame, Event, State>,
  294. ): Route<Body, HttpTransport.HttpPrepared<Frame>>
  295. export function make<Body, Prepared, Frame, Event, State>(
  296. input: MakeInput<Body, Frame, Event, State> | MakeTransportInput<Body, Prepared, Frame, Event, State>,
  297. ): Route<Body, Prepared> | Route<Body, HttpTransport.HttpPrepared<Frame>> {
  298. if ("transport" in input) return makeFromTransport(input)
  299. const protocol = input.protocol
  300. return makeFromTransport({
  301. id: input.id,
  302. provider: input.provider,
  303. protocol,
  304. endpoint: input.endpoint,
  305. auth: input.auth,
  306. headers: input.headers,
  307. transport: HttpTransport.httpJson({ framing: input.framing }),
  308. defaults: input.defaults,
  309. })
  310. }
  311. // `compile` is the important boundary: it turns a common `LLMRequest` into a
  312. // validated provider body plus transport-private prepared data, but does not
  313. // execute transport.
  314. const compile = Effect.fn("LLM.compile")(function* (request: LLMRequest) {
  315. const resolved = applyCachePolicy(resolveRequestOptions(request))
  316. const route = resolved.model.route
  317. const body = yield* route.body
  318. .from(resolved)
  319. .pipe(Effect.flatMap(ProviderShared.validateWith(Schema.decodeUnknownEffect(route.body.schema))))
  320. const prepared = yield* route.prepareTransport(body, resolved)
  321. return {
  322. request: resolved,
  323. route,
  324. body,
  325. prepared,
  326. }
  327. })
  328. const prepareWith = Effect.fn("LLMClient.prepare")(function* (request: LLMRequest) {
  329. const compiled = yield* compile(request)
  330. return new PreparedRequest({
  331. id: compiled.request.id ?? "request",
  332. route: compiled.route.id,
  333. protocol: compiled.route.protocol,
  334. model: compiled.request.model,
  335. body: compiled.body,
  336. metadata: { transport: compiled.route.transport.id },
  337. })
  338. })
  339. const streamRequestWith = (runtime: TransportRuntime) => (request: LLMRequest) =>
  340. Stream.unwrap(
  341. Effect.gen(function* () {
  342. const compiled = yield* compile(request)
  343. return compiled.route.streamPrepared(compiled.prepared, compiled.request, runtime)
  344. }),
  345. )
  346. const generateWith = (stream: Interface["stream"]) =>
  347. Effect.fn("LLM.generate")(function* (request: LLMRequest) {
  348. const state = yield* stream(request).pipe(Stream.runFold(LLMResponse.empty, LLMResponse.reduce))
  349. const response = LLMResponse.complete(state)
  350. if (response) return response
  351. return yield* ProviderShared.eventError(
  352. `${request.model.provider}/${request.model.route.id}`,
  353. "Provider stream ended without a terminal finish event",
  354. )
  355. })
  356. export const prepare = <Body = unknown>(request: LLMRequest) =>
  357. prepareWith(request) as Effect.Effect<PreparedRequestOf<Body>, LLMError>
  358. export function stream(request: LLMRequest): Stream.Stream<LLMEvent, LLMError> {
  359. return Stream.unwrap(
  360. Effect.gen(function* () {
  361. return (yield* Service).stream(request)
  362. }),
  363. ) as Stream.Stream<LLMEvent, LLMError>
  364. }
  365. export function generate(request: LLMRequest): Effect.Effect<LLMResponse, LLMError> {
  366. return Effect.gen(function* () {
  367. return yield* (yield* Service).generate(request)
  368. }) as Effect.Effect<LLMResponse, LLMError>
  369. }
  370. export const streamRequest = (request: LLMRequest) =>
  371. Stream.unwrap(
  372. Effect.gen(function* () {
  373. return (yield* Service).stream(request)
  374. }),
  375. )
  376. export const layer: Layer.Layer<Service, never, RequestExecutor.Service> = Layer.effect(
  377. Service,
  378. Effect.gen(function* () {
  379. const stream = streamRequestWith({
  380. http: yield* RequestExecutor.Service,
  381. webSocket: Option.getOrUndefined(yield* Effect.serviceOption(WebSocketExecutor.Service)),
  382. })
  383. return Service.of({ prepare: prepareWith as Interface["prepare"], stream, generate: generateWith(stream) })
  384. }),
  385. )
  386. export const Route = { make } as const
  387. export const LLMClient = {
  388. Service,
  389. layer,
  390. prepare,
  391. stream,
  392. generate,
  393. } as const